diff --git a/src/Makefile.am b/src/Makefile.am index 7fbb2c8b..c4a53d58 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -62,7 +62,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \ htsname.c htsrobots.c htstools.c htswizard.c \ htsalias.c htsthread.c htsindex.c htsbauth.c \ htsmd5.c htszlib.c htswrap.c htsconcat.c \ - htsmodules.c htscharset.c punycode.c htsencoding.c \ + htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \ md5.c \ minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \ hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \ @@ -70,7 +70,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \ htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \ htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \ htshelp.h htsindex.h htslib.h htsmd5.h \ - htsmodules.h htsname.h htsnet.h \ + htsmodules.h htsname.h htsnet.h htssniff.h \ htsopt.h htsrobots.h htsthread.h \ htstools.h htswizard.h htswrap.h htszlib.h \ htsstrings.h htsarrays.h httrack-library.h \ diff --git a/src/htscache.c b/src/htscache.c index b743a7e4..1c76f4d4 100644 --- a/src/htscache.c +++ b/src/htscache.c @@ -596,15 +596,18 @@ htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr, return cache_readex(opt, cache, adr, fil, save, location, NULL, 1); } -htsblk cache_read_including_broken(httrackp * opt, cache_back * cache, - const char *adr, const char *fil) { - htsblk r = cache_read(opt, cache, adr, fil, NULL, NULL); +htsblk cache_read_including_broken(httrackp *opt, cache_back *cache, + const char *adr, const char *fil, + char *return_save) { + htsblk r = cache_readex(opt, cache, adr, fil, NULL, NULL, return_save, 0); if (r.statuscode == -1) { lien_back *itemback = NULL; if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) { r = itemback->r; + if (return_save != NULL) + strlcpybuff(return_save, itemback->url_sav, HTS_URLMAXSIZE * 2); /* cleanup */ back_clear_entry(itemback); /* delete entry content */ freet(itemback); /* delete item */ diff --git a/src/htscache.h b/src/htscache.h index 1d07137c..068e6350 100644 --- a/src/htscache.h +++ b/src/htscache.h @@ -66,8 +66,11 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr, const char *fil, const char *save, char *location); htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr, const char *fil, const char *save, char *location); -htsblk cache_read_including_broken(httrackp * opt, cache_back * cache, - const char *adr, const char *fil); +/* Like cache_read, but also yields entries whose transfer broke; return_save + (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */ +htsblk cache_read_including_broken(httrackp *opt, cache_back *cache, + const char *adr, const char *fil, + char *return_save); htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr, const char *fil, const char *save, char *location, char *return_save, int readonly); diff --git a/src/htsname.c b/src/htsname.c index 9d605750..fa865380 100644 --- a/src/htsname.c +++ b/src/htsname.c @@ -41,6 +41,10 @@ Please visit our Website: http://www.httrack.com #include "htstools.h" #include "htscharset.h" #include "htsencoding.h" +#include "htssniff.h" +#if HTS_USEZLIB +#include "htszlib.h" +#endif #include #define ADD_STANDARD_PATH \ @@ -140,8 +144,7 @@ static void cleanEndingSpaceOrDot(char *s) { /* Wire Content-Type vs URL extension: a patchable wire type wins over an unspecific ext, the HTS_UNKNOWN_MIME sentinel keeps a specific non-HTML ext - (#267 guard), a declared disagreement is CONTESTED. Sentinel and verdict - ride the cache, so updates stay consistent. */ + (#267 guard), a declared disagreement is CONTESTED (sniffed below). */ typedef enum wire_verdict { WIRE_KEEPS_EXT, WIRE_WINS, @@ -165,8 +168,105 @@ static wire_verdict wire_ext_verdict(httrackp *opt, const char *wiremime, return WIRE_CONTESTED; } -static int wire_patches_ext(httrackp *opt, const char *wiremime, - const char *file) { +/* Optional evidence for a contested wire-vs-ext verdict. */ +typedef struct sniff_src { + struct_back *sback; /* live backing (looked up by adr/fil) */ + const lien_back *headers; /* snapshot: r.adr, else the url_sav file */ + const char *adr, *fil; + const char *prev_save; /* previous run's save name (cache X-Save) */ +} sniff_src; + +#if HTS_USEZLIB +/* Inflate the head of a gzip/zlib stream; 0 when undecodable. */ +static size_t sniff_inflate_head(const void *in, size_t in_len, void *out, + size_t out_len) { + z_stream zs; + size_t n = 0; + int err; + + memset(&zs, 0, sizeof(zs)); + if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */ + return 0; + zs.next_in = (const Bytef *) in; + zs.avail_in = (uInt) in_len; + zs.next_out = (Bytef *) out; + zs.avail_out = (uInt) out_len; + err = inflate(&zs, Z_SYNC_FLUSH); + if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR) + n = out_len - zs.avail_out; + inflateEnd(&zs); + return n; +} +#endif + +static size_t sniff_read_head(const char *path, void *buf, size_t len) { + char catbuff[CATBUFF_SIZE]; + FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "rb"); + size_t n = 0; + + if (fp != NULL) { + n = fread(buf, 1, len, fp); + fclose(fp); + } + return n; +} + +/* Body head of one slot: memory, else its flushed on-disk file (url_sav, or + tmpfile for a compressed stream); inflated so the sniff sees the final body. + */ +static size_t sniff_slot_head(const lien_back *slot, void *buf, size_t len) { + const htsblk *const r = &slot->r; + size_t n = 0; + + if (r->adr != NULL && r->size > 0) { + n = (size_t) r->size < len ? (size_t) r->size : len; + memcpy(buf, r->adr, n); + } else { + if (r->out != NULL) + fflush(r->out); + if (slot->url_sav[0] != '\0') + n = sniff_read_head(slot->url_sav, buf, len); + if (n == 0 && slot->tmpfile != NULL && slot->tmpfile[0] != '\0') + n = sniff_read_head(slot->tmpfile, buf, len); + } + if (n > 0 && r->compressed) { +#if HTS_USEZLIB + unsigned char raw[HTS_SNIFF_LEN]; + + if (n > sizeof(raw)) + n = sizeof(raw); + memcpy(raw, buf, n); + n = sniff_inflate_head(raw, n, buf, len); +#else + n = 0; +#endif + } + return n; +} + +/* Up to len leading body bytes; 0 when unavailable, and always in + non-delayed mode (its HEAD-probe first run couldn't sniff either). */ +static size_t sniff_body_head(httrackp *opt, const sniff_src *src, void *buf, + size_t len) { + size_t n = 0; + + if (src == NULL || opt->savename_delayed == HTS_SAVENAME_DELAYED_NONE) + return 0; + /* live backing slot: a snapshot (back_copy_static) loses r.adr/r.out */ + if (src->sback != NULL && src->adr != NULL && src->fil != NULL) { + const int b = back_index(opt, src->sback, src->adr, src->fil, NULL); + + if (b >= 0) + n = sniff_slot_head(&src->sback->lnk[b], buf, len); + } + if (n == 0 && src->headers != NULL) + n = sniff_slot_head(src->headers, buf, len); + return n; +} + +/* Contested verdicts: magic proving the URL ext keeps it, else wire wins. */ +static int wire_patches_ext(httrackp *opt, const sniff_src *src, + const char *wiremime, const char *file) { char urlmime[256]; switch (wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime))) { @@ -175,22 +275,51 @@ static int wire_patches_ext(httrackp *opt, const char *wiremime, case WIRE_WINS: return 1; case WIRE_CONTESTED: - break; /* no content evidence is consulted today: trust the wire */ + break; + } + if (src != NULL) { + if (hts_sniff_mime_known(urlmime)) { + unsigned char head[HTS_SNIFF_LEN]; + const size_t n = sniff_body_head(opt, src, head, sizeof(head)); + + if (n > 0) + return hts_sniff_mime_consistent(head, n, urlmime) ? 0 : 1; + } + /* no bytes: reproduce the previous run's verdict (cached X-Save name) */ + if (src->prev_save != NULL && src->prev_save[0] != '\0') { + char prevmime[256]; + + prevmime[0] = '\0'; + if (get_httptype_sized(opt, prevmime, sizeof(prevmime), src->prev_save, + 0) && + strfield2(prevmime, urlmime)) + return 0; + } } return 1; } +int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime, + const char *file) { + char urlmime[256]; + + return wiremime != NULL && strnotempty(wiremime) && + wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime)) == + WIRE_CONTESTED && + hts_sniff_mime_known(urlmime); +} + /* Wire-metadata name change: a Content-Disposition filename wins (returns 2), else the declared type's ext when wire_patches_ext() allows (returns 1), else 0. ext receives the new extension or replacement filename. */ -static int resolve_extension(httrackp *opt, const char *cdispo, - const char *contenttype, const char *fil, - char *ext, size_t ext_size) { +static int resolve_extension(httrackp *opt, const sniff_src *src, + const char *cdispo, const char *contenttype, + const char *fil, char *ext, size_t ext_size) { if (strnotempty(cdispo)) { strlcpybuff(ext, cdispo, ext_size); return 2; } - if (wire_patches_ext(opt, contenttype, fil) && + if (wire_patches_ext(opt, src, contenttype, fil) && give_mimext(ext, ext_size, contenttype)) return 1; return 0; @@ -442,14 +571,21 @@ int url_savename(lien_adrfilsave *const afs, if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD || ishtml(opt, fil) < 0) { // unsure whether it's html or a file // lire dans le cache - htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement + char BIGSTK previous_save[HTS_URLMAXSIZE * 2]; + htsblk r; + + previous_save[0] = '\0'; + r = cache_read_including_broken(opt, cache, adr, fil, + previous_save); // test uniquement if (r.statuscode != -1) { // cache entry read OK hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s", adr_complete, fil_complete); if (!HTTP_IS_REDIRECT(r.statuscode)) { - ext_chg = resolve_extension(opt, r.cdispo, r.contenttype, fil, - ext, sizeof(ext)); + const sniff_src src = {sback, NULL, adr, fil, previous_save}; + + ext_chg = resolve_extension(opt, &src, r.cdispo, r.contenttype, + fil, ext, sizeof(ext)); } } else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD && is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER. @@ -476,7 +612,9 @@ int url_savename(lien_adrfilsave *const afs, !opt->state.stop) { // Check if the file is ready in backing. if (headers != NULL && headers->status >= 0 && !is_redirect) { - ext_chg = resolve_extension(opt, headers->r.cdispo, + const sniff_src src = {sback, headers, adr, fil, NULL}; + + ext_chg = resolve_extension(opt, &src, headers->r.cdispo, headers->r.contenttype, headers->url_fil, ext, sizeof(ext)); } @@ -687,7 +825,7 @@ int url_savename(lien_adrfilsave *const afs, // no error: change the type? ext_chg = resolve_extension( - opt, back[b].r.cdispo, back[b].r.contenttype, + opt, NULL, back[b].r.cdispo, back[b].r.contenttype, back[b].url_fil, ext, sizeof(ext)); } // FIN Si non déplacé, forcer type? diff --git a/src/htsname.h b/src/htsname.h index e8fea466..f9fe5d5d 100644 --- a/src/htsname.h +++ b/src/htsname.h @@ -100,6 +100,8 @@ void standard_name(char *b, size_t bsize, const char *dot_pos, const char *nom_pos, const char *fil_complete, int short_ver); void url_savename_addstr(char *d, const char *s); +/* Contested wire-vs-ext verdict that a body sniff could settle (htssniff.h). */ +int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime, const char *file); char *url_md5(char *digest_buffer, const char *fil_complete); void url_savename_refname(const char *adr, const char *fil, char *filename); char *url_savename_refname_fullpath(httrackp * opt, const char *adr, diff --git a/src/htsparse.c b/src/htsparse.c index cdb2593f..c1d645c2 100644 --- a/src/htsparse.c +++ b/src/htsparse.c @@ -49,6 +49,7 @@ Please visit our Website: http://www.httrack.com #include "htsindex.h" #include "htscharset.h" #include "htsencoding.h" +#include "htssniff.h" /* external modules */ #include "htsmodules.h" @@ -4695,22 +4696,26 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs, if (!RUN_CALLBACK7 (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot, (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) { + back_set_unlocked(sback, b); return -1; } else if (opt->state._hts_cancel || !back_checkmirror(opt)) { // cancel 2 ou 1 (cancel parsing) back_delete(opt, cache, sback, b); // cancel test break; } } - } while( - /* dns/connect/request */ - (back[b].status >= 99 && back[b].status <= 101) - || - /* For redirects, wait for request to be terminated */ - (HTTP_IS_REDIRECT(back[b].r.statuscode) && back[b].status > 0) - || - /* Same for errors */ - (HTTP_IS_ERROR(back[b].r.statuscode) && back[b].status > 0) - ); + } while ( + /* dns/connect/request */ + (back[b].status >= 99 && back[b].status <= 101) || + /* For redirects, wait for request to be terminated */ + (HTTP_IS_REDIRECT(back[b].r.statuscode) && back[b].status > 0) || + /* Same for errors */ + (HTTP_IS_ERROR(back[b].r.statuscode) && back[b].status > 0) || + /* Contested type: wait for a sniffable body head (or EOF) */ + (back[b].r.statuscode == HTTP_OK && back[b].status > 0 && + strnotempty(back[b].r.cdispo) == 0 && + back[b].r.size < HTS_SNIFF_LEN && + hts_ext_sniff_wanted(opt, back[b].r.contenttype, + back[b].url_fil))); if (b >= 0) { back_set_unlocked(sback, b); // Unlocked entry } @@ -4845,8 +4850,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs, /* Still have a back reference */ if (b >= 0) { - /* Patch destination filename for direct-to-disk mode, BEFORE any - finalize: it records and caches the entry under url_sav */ + /* patch url_sav BEFORE finalize: it records/caches under this name + */ strcpybuff(back[b].url_sav, afs->save); /* Finalize now as we have the type */ if (back[b].status == STATUS_READY) { diff --git a/src/htsselftest.c b/src/htsselftest.c index 06d211c1..46258a42 100644 --- a/src/htsselftest.c +++ b/src/htsselftest.c @@ -52,6 +52,7 @@ Please visit our Website: http://www.httrack.com #include "htsencoding.h" #include "htsftp.h" #include "htsmd5.h" +#include "htssniff.h" #if HTS_USEZLIB #include "htszlib.h" #endif @@ -1141,6 +1142,22 @@ static size_t st_decode_body(const char *arg, char *buf, size_t size) { return n; } +static int st_sniff(httrackp *opt, int argc, char **argv) { + char BIGSTK body[1024]; + size_t n; + + (void) opt; + if (argc < 2) { + fprintf(stderr, "sniff: needs a content-type and a body\n"); + return 1; + } + n = st_decode_body(argv[1], body, sizeof(body)); + printf("sniff: known=%d consistent=%d\n", + hts_sniff_mime_known(argv[0]) == HTS_TRUE, + hts_sniff_mime_consistent(body, n, argv[0]) == HTS_TRUE); + return 0; +} + static int st_savename(httrackp *opt, int argc, char **argv) { lien_adrfilsave afs; cache_back cache; @@ -1211,7 +1228,7 @@ static int st_savename(httrackp *opt, int argc, char **argv) { } *sep = '\0'; /* one-entry cache in cwd, reopened read-only; body is PNG magic on - purpose: naming must not depend on stored content */ + purpose: only the recorded name (X-Save) may drive the naming */ StringCopy(opt->path_log, ""); cache.type = 1; cache.log = cache.errlog = stderr; @@ -1272,7 +1289,7 @@ static int st_savename(httrackp *opt, int argc, char **argv) { if (cdispo != NULL) strcpybuff(headers.r.cdispo, cdispo); strcpybuff(headers.url_fil, argv[0]); - if (body != NULL) { /* leading body bytes, exposed via url_sav */ + if (body != NULL) { /* leading body bytes, read via url_sav */ char BIGSTK data[1024]; const size_t n = st_decode_body(body, data, sizeof(data)); FILE *const fp = fopen(bodyfile, "wb"); @@ -2095,6 +2112,8 @@ static const struct selftest_entry { st_header}, {"savename", " [key=value ...]", "local save-name for a URL", st_savename}, + {"sniff", " ", "MIME magic consistency", + st_sniff}, {"cache", "", "cache read/write round-trip self-test", st_cache}, {"cache-golden", " [regen]", "frozen cache-format read self-test", st_cache_golden}, diff --git a/src/htssniff.c b/src/htssniff.c new file mode 100644 index 00000000..ae7a6580 --- /dev/null +++ b/src/htssniff.c @@ -0,0 +1,352 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) 1998-2017 Xavier Roche and other contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Important notes: + +- We hereby ask people using this source NOT to use it in purpose of grabbing +emails addresses, or collecting any other private information on persons. +This would disgrace our work, and spoil the many hours we spent on it. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: MIME magic-byte consistency checks */ +/* Author: Xavier Roche */ +/* ------------------------------------------------------------ */ + +#include "htssniff.h" + +#include + +#include "htslib.h" + +/* One magic rule: `len` bytes at `off` confirm `mime`. */ +typedef struct sniff_magic { + const char *mime; + unsigned short off; + unsigned char len; + const char *bytes; +} sniff_magic; + +/* Direction is mime -> magic (verify a claim, never classify); types with + no reliable magic (plain text, css, js..) are deliberately absent. Patterns + follow the WHATWG MIME Sniffing Standard tables where it defines them + (https://mimesniff.spec.whatwg.org/); the rest covers httrack's wider MIME + set. Spec-only types absent from our MIME tables (EOT, font/collection) + are omitted as unreachable. */ +static const sniff_magic sniff_table[] = { + /* images */ + {"image/jpeg", 0, 3, "\xff\xd8\xff"}, + {"image/pipeg", 0, 3, "\xff\xd8\xff"}, + {"image/pjpeg", 0, 3, "\xff\xd8\xff"}, + {"image/png", 0, 8, "\x89PNG\r\n\x1a\n"}, + {"image/gif", 0, 6, "GIF87a"}, + {"image/gif", 0, 6, "GIF89a"}, + {"image/bmp", 0, 2, "BM"}, + {"image/tiff", 0, 4, "II*\0"}, + {"image/tiff", 0, 4, "MM\0*"}, + {"image/x-icon", 0, 4, "\0\0\1\0"}, + {"image/x-icon", 0, 4, "\0\0\2\0"}, /* Windows cursor, per the spec */ + {"image/x-portable-bitmap", 0, 2, "P1"}, + {"image/x-portable-bitmap", 0, 2, "P4"}, + {"image/x-portable-pixmap", 0, 2, "P3"}, + {"image/x-portable-pixmap", 0, 2, "P6"}, + {"image/x-xpixmap", 0, 9, "/* XPM */"}, + {"image/x-xbitmap", 0, 7, "#define"}, + {"image/x-rgb", 0, 2, "\x01\xda"}, + {"image/x-cmu-raster", 0, 4, "\xf1\x00\x40\xbb"}, + /* audio */ + {"audio/mpeg", 0, 3, "ID3"}, + {"audio/basic", 0, 4, ".snd"}, + {"audio/mid", 0, 8, "MThd\0\0\0\6"}, + {"audio/midi", 0, 8, "MThd\0\0\0\6"}, + {"audio/x-pn-realaudio", 0, 4, ".ra\xfd"}, + {"audio/x-pn-realaudio", 0, 4, ".RMF"}, + {"audio/x-pn-realaudio-plugin", 0, 4, ".ra\xfd"}, + {"audio/x-pn-realaudio-plugin", 0, 4, ".RMF"}, + {"audio/flac", 0, 4, "fLaC"}, + {"audio/aac", 0, 4, "ADIF"}, + /* video */ + {"video/mpeg", 0, 4, "\x00\x00\x01\xba"}, + {"video/mpeg", 0, 4, "\x00\x00\x01\xb3"}, + {"video/x-sgi-movie", 0, 4, "MOVI"}, + /* archives / compression */ + {"application/x-gzip", 0, 3, "\x1f\x8b\x08"}, + {"multipart/x-gzip", 0, 3, "\x1f\x8b\x08"}, + {"application/x-compressed", 0, 3, "\x1f\x8b\x08"}, + {"application/x-compress", 0, 2, "\x1f\x9d"}, + {"application/x-bzip2", 0, 3, "BZh"}, + {"application/x-7z-compressed", 0, 6, "7z\xbc\xaf\x27\x1c"}, + /* 6-byte prefix common to RAR4 (spec) and RAR5 */ + {"application/x-rar-compressed", 0, 6, "Rar!\x1a\x07"}, + {"application/zstd", 0, 4, "\x28\xb5\x2f\xfd"}, + {"application/arj", 0, 2, "\x60\xea"}, + {"application/x-cpio", 0, 6, "070701"}, + {"application/x-cpio", 0, 6, "070707"}, + {"application/x-cpio", 0, 2, "\xc7\x71"}, + {"application/x-sv4cpio", 0, 6, "070701"}, + {"application/x-sv4crc", 0, 6, "070702"}, + {"application/x-stuffit", 0, 8, "StuffIt "}, + {"application/x-stuffit", 0, 4, "SIT!"}, + {"application/mac-binhex40", 0, 10, "(This file"}, + /* documents */ + {"application/pdf", 0, 5, "%PDF-"}, + {"application/postscript", 0, 2, "%!"}, + {"application/rtf", 0, 5, "{\\rtf"}, + {"application/x-dvi", 0, 2, "\xf7\x02"}, + {"application/x-hdf", 0, 4, "\x0e\x03\x13\x01"}, + {"application/x-hdf", 0, 8, "\x89HDF\r\n\x1a\n"}, + {"application/x-netcdf", 0, 4, "CDF\x01"}, + {"application/x-netcdf", 0, 4, "CDF\x02"}, + {"application/x-msaccess", 0, 19, "\0\1\0\0Standard Jet DB"}, + /* fonts */ + {"font/woff", 0, 4, "wOFF"}, + {"font/woff2", 0, 4, "wOF2"}, + {"font/ttf", 0, 4, "\0\1\0\0"}, + {"font/ttf", 0, 4, "true"}, + {"font/otf", 0, 4, "OTTO"}, + /* misc */ + {"application/x-shockwave-flash", 0, 3, "FWS"}, + {"application/x-shockwave-flash", 0, 3, "CWS"}, + {"application/x-shockwave-flash", 0, 3, "ZWS"}, + {"application/futuresplash", 0, 3, "FWS"}, + {"application/x-director", 0, 4, "RIFX"}, + {"application/x-director", 0, 4, "XFIR"}, + {"application/x-java-vm", 0, 4, "\xca\xfe\xba\xbe"}, + {"application/wasm", 0, 4, "\0asm"}, + {"application/x-msmetafile", 0, 4, "\xd7\xcd\xc6\x9a"}, + {"application/x-msmetafile", 0, 4, "\x01\x00\x09\x00"}, + {"application/x-x509-ca-cert", 0, 2, "\x30\x82"}, + {"application/x-pkcs12", 0, 2, "\x30\x82"}, + {"application/x-pkcs7-mime", 0, 2, "\x30\x82"}, + {"application/x-pkcs7-signature", 0, 2, "\x30\x82"}, + {"application/x-pkcs7-certificates", 0, 2, "\x30\x82"}, + {"x-world/x-vrml", 0, 5, "#VRML"}, + {"application/x-bittorrent", 0, 11, "d8:announce"}, + {"drawing/x-dwf", 0, 4, "(DWF"}, + {"application/acad", 0, 4, "AC10"}, + {NULL, 0, 0, NULL}}; + +/* MIME families sharing a container magic */ +static const char *const zip_mimes[] = { + "application/zip", "application/x-zip-compressed", "multipart/x-zip", NULL}; +static const char *const zip_mime_prefixes[] = { + "application/vnd.openxmlformats-officedocument.", + "application/vnd.oasis.opendocument.", NULL}; +static const char *const ole_mimes[] = {"application/msword", + "application/excel", + "application/vnd.ms-excel", + "application/powerpoint", + "application/vnd.ms-powerpoint", + "application/vnd.ms-project", + "application/vnd.ms-works", + "application/x-msmoney", + "application/x-mspublisher", + NULL}; +static const char *const tar_mimes[] = { + "application/x-tar", "application/x-ustar", "application/x-gtar", NULL}; +static const char *const ogg_mimes[] = {"application/ogg", "audio/ogg", + "video/ogg", "audio/opus", NULL}; +static const char *const ebml_mimes[] = {"video/webm", "audio/webm", NULL}; +/* ISO-BMFF, any 'ftyp' brand: containers overlap too much to split */ +static const char *const bmff_mimes[] = {"video/mp4", "audio/mp4", + "video/quicktime", NULL}; +static const char *const avif_mimes[] = {"image/avif", NULL}; +static const char *const heic_mimes[] = {"image/heic", NULL}; +static const char *const asf_mimes[] = {"video/x-ms-asf", "video/x-ms-wmv", + "video/x-la-asf", NULL}; +static const char *const xml_mimes[] = {"application/xml", "text/xml", + "image/svg+xml", "image/svg-xml", NULL}; +static const char *const svg_mimes[] = {"image/svg+xml", "image/svg-xml", NULL}; +static const char *const html_mimes[] = {"text/html", NULL}; +static const char *const pem_mimes[] = { + "application/x-x509-ca-cert", "application/x-pkcs7-certificates", + "application/x-pkcs7-mime", "application/x-pkcs7-signature", NULL}; + +static hts_boolean mime_in(const char *const *list, const char *mime) { + size_t i; + + for (i = 0; list[i] != NULL; i++) + if (strfield2(list[i], mime)) + return HTS_TRUE; + return HTS_FALSE; +} + +static hts_boolean mime_in_prefix(const char *const *list, const char *mime) { + size_t i; + + for (i = 0; list[i] != NULL; i++) + if (strfield(mime, list[i])) + return HTS_TRUE; + return HTS_FALSE; +} + +static hts_boolean has_bytes(const unsigned char *d, size_t n, size_t off, + const char *bytes, size_t len) { + /* overflow-safe: untrusted n alone on one side */ + return n >= off && len <= n - off && memcmp(d + off, bytes, len) == 0 + ? HTS_TRUE + : HTS_FALSE; +} + +static unsigned char ascii_lower(unsigned char c) { + return c >= 'A' && c <= 'Z' ? (unsigned char) (c + 32) : c; +} + +/* Case-insensitive text prefix after an optional UTF-8 BOM and whitespace. */ +static hts_boolean has_text_prefix(const unsigned char *d, size_t n, + const char *prefix) { + const size_t len = strlen(prefix); + size_t i, k; + + i = n >= 3 && memcmp(d, "\xef\xbb\xbf", 3) == 0 ? 3 : 0; + while (i < n && (d[i] == ' ' || d[i] == '\t' || d[i] == '\r' || d[i] == '\n')) + i++; + if (len > n - i) /* i <= n from the loop above */ + return HTS_FALSE; + for (k = 0; k < len; k++) + if (ascii_lower(d[i + k]) != ascii_lower((unsigned char) prefix[k])) + return HTS_FALSE; + return HTS_TRUE; +} + +typedef enum sniff_op { + SNIFF_QUERY_KNOWN, /* is any rule defined for this MIME? */ + SNIFF_QUERY_MATCH /* do the bytes confirm this MIME? */ +} sniff_op; + +/* Single walk for both queries so the rule set can't drift apart. */ +static hts_boolean sniff_eval(sniff_op op, const unsigned char *d, size_t n, + const char *mime) { + size_t i; + +/* KNOWN short-circuits; MATCH tests the magic */ +#define SNIFF_RULE(cond) \ + do { \ + if (op == SNIFF_QUERY_KNOWN) \ + return HTS_TRUE; \ + if (cond) \ + return HTS_TRUE; \ + } while (0) + + for (i = 0; sniff_table[i].mime != NULL; i++) { + if (strfield2(sniff_table[i].mime, mime)) { + SNIFF_RULE(has_bytes(d, n, sniff_table[i].off, sniff_table[i].bytes, + sniff_table[i].len)); + } + } + if (mime_in(zip_mimes, mime) || mime_in_prefix(zip_mime_prefixes, mime)) { + SNIFF_RULE(has_bytes(d, n, 0, "PK\3\4", 4) || + has_bytes(d, n, 0, "PK\5\6", 4)); + } + if (mime_in(ole_mimes, mime)) { + SNIFF_RULE(has_bytes(d, n, 0, "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", 8)); + } + if (mime_in(tar_mimes, mime)) { + SNIFF_RULE(has_bytes(d, n, 257, "ustar", 5)); + } + if (mime_in(ogg_mimes, mime)) { + SNIFF_RULE(has_bytes(d, n, 0, "OggS\0", 5)); + } + if (mime_in(ebml_mimes, mime)) { + SNIFF_RULE(has_bytes(d, n, 0, "\x1a\x45\xdf\xa3", 4)); + } + if (mime_in(bmff_mimes, mime)) { + SNIFF_RULE(has_bytes(d, n, 4, "ftyp", 4)); + } + if (mime_in(avif_mimes, mime)) { + SNIFF_RULE(has_bytes(d, n, 4, "ftypavif", 8) || + has_bytes(d, n, 4, "ftypavis", 8)); + } + if (mime_in(heic_mimes, mime)) { + SNIFF_RULE( + has_bytes(d, n, 4, "ftyphei", 7) || has_bytes(d, n, 4, "ftyphev", 7) || + has_bytes(d, n, 4, "ftypmif1", 8) || has_bytes(d, n, 4, "ftypmsf1", 8)); + } + if (mime_in(asf_mimes, mime)) { + SNIFF_RULE(has_bytes(d, n, 0, "\x30\x26\xb2\x75\x8e\x66\xcf\x11", 8)); + } + if (strfield2("audio/x-wav", mime)) { + SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && has_bytes(d, n, 8, "WAVE", 4)); + } + if (strfield2("video/x-msvideo", mime)) { + SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && has_bytes(d, n, 8, "AVI ", 4)); + } + if (strfield2("image/webp", mime)) { + SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && + has_bytes(d, n, 8, "WEBPVP", 6)); + } + if (strfield2("image/x-portable-anymap", mime)) { + SNIFF_RULE(n >= 2 && d[0] == 'P' && d[1] >= '1' && d[1] <= '6'); + } + if (strfield2("audio/x-aiff", mime)) { + SNIFF_RULE( + has_bytes(d, n, 0, "FORM", 4) && + (has_bytes(d, n, 8, "AIFF", 4) || has_bytes(d, n, 8, "AIFC", 4))); + } + if (strfield2("audio/mpeg", mime)) { + /* MPEG audio frame sync (11 bits), valid layer and bitrate fields */ + SNIFF_RULE(n >= 2 && d[0] == 0xff && (d[1] & 0xe0) == 0xe0 && + (d[1] & 0x06) != 0); + } + if (strfield2("audio/aac", mime)) { + /* ADTS sync */ + SNIFF_RULE(n >= 2 && d[0] == 0xff && (d[1] & 0xf6) == 0xf0); + } + if (strfield2("video/mp2t", mime)) { + SNIFF_RULE(n >= 1 && d[0] == 0x47 && (n <= 188 || d[188] == 0x47)); + } + if (mime_in(xml_mimes, mime)) { + SNIFF_RULE(has_text_prefix(d, n, ". + +Important notes: + +- We hereby ask people using this source NOT to use it in purpose of grabbing +emails addresses, or collecting any other private information on persons. +This would disgrace our work, and spoil the many hours we spent on it. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: MIME magic-byte consistency checks */ +/* Author: Xavier Roche */ +/* ------------------------------------------------------------ */ + +#ifndef HTSSNIFF_DEFH +#define HTSSNIFF_DEFH + +#include +#include "htsglobal.h" + +/* Leading-body window read to arbitrate a wire/extension MIME conflict. */ +#define HTS_SNIFF_LEN 512 + +/* Can a magic rule ever confirm this MIME? (whether sniffing is worth it) */ +hts_boolean hts_sniff_mime_known(const char *mime); + +/* TRUE when the leading body bytes are consistent with the claimed MIME; + FALSE on unknown MIME, unknown magic, or too-short data (fail-safe). */ +hts_boolean hts_sniff_mime_consistent(const void *data, size_t size, + const char *mime); + +#endif diff --git a/tests/01_engine-savename.test b/tests/01_engine-savename.test index b5d672fc..3dfdc045 100755 --- a/tests/01_engine-savename.test +++ b/tests/01_engine-savename.test @@ -81,15 +81,14 @@ name '/x.pdf' 'text/html' 'x.html' status=-1 name '/x.html' 'text/html' 'x.html' status=-1 name '/x.php' 'application/pdf' 'x.pdf' status=-1 cdispo=report.pdf -# Contested type (wire disagrees with a specific ext): the wire is trusted and -# body bytes are not consulted; pinned so a content-based tie-break shows up -# as an explicit flip of these rows. -name '/photo.jpg' 'image/png' 'photo.png' body=hex:FFD8FFE000104A46 +# Contested type (wire disagrees with a specific ext): magic bytes proving the +# extension right keep it, anything else trusts the wire as before. +name '/photo.jpg' 'image/png' 'photo.jpg' body=hex:FFD8FFE000104A46 name '/photo.jpg' 'image/png' 'photo.png' body=hex:89504E470D0A1A0A name '/photo.jpg' 'image/png' 'photo.png' -name '/doc.pdf' 'text/html' 'doc.html' body=hex:255044462D312E34 +name '/doc.pdf' 'text/html' 'doc.pdf' body=hex:255044462D312E34 name '/doc.pdf' 'text/html' 'doc.html' 'body=soft 404' -name '/style.css' 'image/png' 'style.png' 'body=body { }' +name '/style.css' 'image/png' 'style.png' 'body=body { }' # no rule for css: wire wins # A redirect answer resolves nothing: delayed placeholder name. name '/x.php' 'text/html' 'x.0.delayed' statuscode=301 diff --git a/tests/01_engine-sniff.test b/tests/01_engine-sniff.test new file mode 100644 index 00000000..15d0e624 --- /dev/null +++ b/tests/01_engine-sniff.test @@ -0,0 +1,87 @@ +#!/bin/bash +# + +set -euo pipefail + +# MIME magic consistency (-#test=sniff ), the +# tie-break behind htsname's wire-vs-extension naming. + +chk() { + local mime="$1" body="$2" want="$3" + out="$(httrack -#test=sniff "$mime" "$body" | sed -n 's/^sniff: //p')" + test "$out" == "$want" || { + echo "FAIL: '$mime' '$body' -> '$out' (want '$want')" + exit 1 + } +} + +yes='known=1 consistent=1' +no='known=1 consistent=0' +unk='known=0 consistent=0' + +# images +chk image/jpeg hex:FFD8FFE000104A46 "$yes" +chk image/png hex:89504E470D0A1A0A "$yes" +chk image/png hex:FFD8FFE000104A46 "$no" # jpeg bytes are not a png +chk image/gif 'GIF89a' "$yes" +chk image/bmp 'BMxxxx' "$yes" +chk image/tiff hex:49492A00 "$yes" +chk image/tiff hex:4D4D002A "$yes" # both endians +chk image/x-icon hex:00000100 "$yes" +chk image/x-icon hex:00000200 "$yes" # Windows cursor, spec maps to x-icon +chk image/webp 'RIFFxxxxWEBPVP' "$yes" +chk image/webp 'RIFFxxxxWAVE' "$no" # riff subtype discriminates +chk image/avif hex:0000001C6674797061766966 "$yes" +chk image/avif hex:0000001C6674797068656963 "$no" # heic brand is not avif +chk image/heic hex:0000001C6674797068656963 "$yes" +chk image/svg+xml '' "$yes" +chk image/svg+xml $'\xef\xbb\xbf ' "$yes" # BOM+ws skip + +# audio / video +chk audio/mpeg 'ID3xxx' "$yes" +chk audio/mpeg hex:FFFB9000 "$yes" # bare frame sync +chk audio/aac hex:FFF15080 "$yes" +chk audio/flac 'fLaC' "$yes" +chk audio/ogg hex:4F67675300 "$yes" +chk audio/x-wav 'RIFFxxxxWAVE' "$yes" +chk video/x-msvideo 'RIFFxxxxAVI ' "$yes" +chk video/x-msvideo 'RIFFxxxxWAVE' "$no" +chk video/mp4 hex:000000186674797069736F6D "$yes" +chk video/webm hex:1A45DFA3 "$yes" +chk video/mpeg hex:000001BA "$yes" +chk video/x-ms-wmv hex:3026B2758E66CF11 "$yes" + +# archives; zip magic covers the office-container families +chk application/zip hex:504B0304 "$yes" +chk application/vnd.openxmlformats-officedocument.wordprocessingml.document hex:504B0304 "$yes" +chk application/vnd.oasis.opendocument.text hex:504B0304 "$yes" +chk application/msword hex:D0CF11E0A1B11AE1 "$yes" +chk application/msword hex:504B0304 "$no" # legacy .doc is OLE, not zip +chk application/x-gzip hex:1F8B08 "$yes" +chk application/x-bzip2 'BZh9' "$yes" +chk application/x-7z-compressed hex:377ABCAF271C "$yes" +chk application/x-rar-compressed hex:526172211A07 "$yes" +chk application/zstd hex:28B52FFD "$yes" +chk application/x-tar "hex:$(printf '00%.0s' {1..257})7573746172" "$yes" # ustar at 257 +chk application/x-tar hex:7573746172 "$no" + +# documents, fonts, misc +chk application/pdf '%PDF-1.7' "$yes" +chk application/pdf 'soft 404' "$no" +chk application/postscript '%!PS-Adobe' "$yes" +chk application/rtf '{\rtf1' "$yes" +chk font/woff2 'wOF2' "$yes" +chk font/otf 'OTTO' "$yes" +chk font/ttf hex:0001000000 "$yes" +chk application/x-shockwave-flash 'CWSx' "$yes" +chk application/x-java-vm hex:CAFEBABE "$yes" +chk application/wasm hex:0061736D "$yes" +chk text/html $' \r\n' "$yes" +chk text/html '' "$yes" +chk text/html 'plain text, no markup' "$no" +chk text/xml '' "$yes" + +# no magic rule at all: never confirmed, never blocks the wire type +chk text/css 'body { }' "$unk" +chk text/plain 'hello' "$unk" +chk application/x-javascript 'var x;' "$unk" diff --git a/tests/01_zlib-savename-cached.test b/tests/01_zlib-savename-cached.test index 365e3e8c..ec4c1342 100644 --- a/tests/01_zlib-savename-cached.test +++ b/tests/01_zlib-savename-cached.test @@ -23,11 +23,11 @@ name() { } } -# Names are re-derived from the stored headers on every run: neither the -# recorded save name nor the cached body bytes change the verdict (pinned). -name '/photo.jpg' 'image/png' 'photo.png' 'cached=image/png|www.example.com/photo.jpg' +# No live bytes: the recorded save name (X-Save) reproduces the previous +# verdict; cached body bytes (PNG magic) are ignored; css has no magic rule. +name '/photo.jpg' 'image/png' 'photo.jpg' 'cached=image/png|www.example.com/photo.jpg' name '/photo.jpg' 'image/png' 'photo.png' 'cached=image/png|www.example.com/photo.png' name '/photo.jpg' 'image/jpeg' 'photo.jpg' 'cached=image/jpeg|www.example.com/photo.png' -name '/style.css' 'image/png' 'style.png' 'cached=image/png|www.example.com/style.css' +name '/style.css' 'image/png' 'style.css' 'cached=image/png|www.example.com/style.css' # agreement keeps the URL ext verbatim (.jpeg), never canonicalized to .jpg name '/photo.jpeg' 'image/jpeg' 'photo.jpeg' 'cached=image/jpeg|www.example.com/photo.jpeg' diff --git a/tests/15_local-types.test b/tests/15_local-types.test index be425087..87a9fd6a 100644 --- a/tests/15_local-types.test +++ b/tests/15_local-types.test @@ -1,11 +1,10 @@ #!/bin/bash # -# Content-Type vs URL-extension naming (issue #267 family) under the default -# delayed type check (-%N2). Policy: a MISSING Content-Type must not clobber a -# URL extension that maps to a specific non-HTML type (.png/.pdf stay as-is); -# an explicitly DECLARED type is trusted, so a binary-looking URL that really -# serves HTML (text/html on .pdf/.jpg) is named .html. The "wrong" names are -# asserted absent so a regression in either direction fails here. +# Content-Type vs URL-extension naming (#267 family, default -%N2). A MISSING +# type keeps a specific non-HTML ext; a DECLARED disagreeing type is trusted +# unless magic bytes prove the ext right (lie/wrongtype/packed keep theirs), +# so a real HTML body (report.pdf) still becomes .html. Wrong names are +# asserted absent so a regression in either direction fails. : "${top_srcdir:=..}" @@ -14,11 +13,11 @@ bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \ --found 'types/notype.pdf' --not-found 'types/notype.html' \ --found 'types/photo.png' \ --found 'types/doc.pdf' \ - --found 'types/lie.html' --not-found 'types/lie.png' \ - --found 'types/wrongtype.png' --not-found 'types/wrongtype.jpg' \ - --found 'types/bigtype.png' --not-found 'types/bigtype.jpg' \ - --found 'types/packed.png' --not-found 'types/packed.jpg' \ - --found 'types/mutant.png' --not-found 'types/mutant.jpg' \ + --found 'types/lie.png' --not-found 'types/lie.html' \ + --found 'types/wrongtype.jpg' --not-found 'types/wrongtype.png' \ + --found 'types/bigtype.jpg' --not-found 'types/bigtype.png' \ + --found 'types/mutant.jpg' --not-found 'types/mutant.png' \ + --found 'types/packed.jpg' --not-found 'types/packed.png' \ --found 'types/report.html' --not-found 'types/report.pdf' \ --found 'types/page.htm' --not-found 'types/page.html' \ --found 'types/script.js' \ diff --git a/tests/18_local-update.test b/tests/18_local-update.test index bcf6ce8d..3c7edd33 100644 --- a/tests/18_local-update.test +++ b/tests/18_local-update.test @@ -1,18 +1,18 @@ #!/bin/bash # -# A second (update) pass must keep the names the first crawl chose. The stored -# Content-Type rides the cache, so the update reads back the same value -- the -# unknown/unknown sentinel for a typeless response, the declared type otherwise -# -- and names consistently: a declared-text/html .pdf stays .html and a -# typeless .png stays .png across the update rather than reverting. +# An update pass keeps the names the first crawl chose: type and save name +# ride the cache, so a declared-text/html .pdf stays .html, a typeless .png +# stays .png, and a sniff-kept ext is reproduced from X-Save even when the +# refetched content changed (mutant.jpg serves PNG bytes on the rerun). : "${top_srcdir:=..}" bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \ --found 'types/report.html' --not-found 'types/report.pdf' \ --found 'types/notype.png' --not-found 'types/notype.html' \ - --found 'types/lie.html' \ - --found 'types/wrongtype.png' --not-found 'types/wrongtype.jpg' \ - --found 'types/packed.png' --not-found 'types/packed.jpg' \ - --found 'types/mutant.png' --not-found 'types/mutant.jpg' \ + --found 'types/lie.png' --not-found 'types/lie.html' \ + --found 'types/wrongtype.jpg' --not-found 'types/wrongtype.png' \ + --found 'types/bigtype.jpg' --not-found 'types/bigtype.png' \ + --found 'types/packed.jpg' --not-found 'types/packed.png' \ + --found 'types/mutant.jpg' --not-found 'types/mutant.png' \ httrack 'BASEURL/types/index.html' diff --git a/tests/Makefile.am b/tests/Makefile.am index 460fd4dd..b309726b 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -54,6 +54,7 @@ TESTS = \ 01_engine-savename.test \ 01_engine-selftest-dispatch.test \ 01_engine-simplify.test \ + 01_engine-sniff.test \ 01_engine-status.test \ 01_engine-stripquery.test \ 01_engine-strsafe.test \