3 // Copyright (C) 2005 Enrico Scholz <enrico.scholz@informatik.tu-chemnitz.de>
5 // This program is free software; you can redistribute it and/or modify
6 // it under the terms of the GNU General Public License as published by
7 // the Free Software Foundation; version 2 of the License.
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 #include "lib/internal.h"
27 #include "lib_internal/matchlist.h"
28 #include "lib_internal/unify.h"
29 #include "ensc_vector/vector.h"
31 #include <beecrypt/beecrypt.h>
47 #define ENSC_WRAPPERS_STDLIB 1
48 #define ENSC_WRAPPERS_UNISTD 1
49 #define ENSC_WRAPPERS_FCNTL 1
50 #define ENSC_WRAPPERS_DIRENT 1
51 #define ENSC_WRAPPERS_IO 1
55 #define HASH_BLOCKSIZE 0x10000000u
56 #define HASH_MINSIZE 0x10
57 #define HASH_MAXBITS 256 // we have to take care about
58 // max filename-length...
61 # error HASH_MINSIZE must be not '0'
65 #define CMD_HELP 0x8000
66 #define CMD_VERSION 0x8001
68 #define CMD_DESTINATION 0x1000
69 #define CMD_INSECURE 0x1001
70 #define CMD_SLEDGE 0x1002
71 #define CMD_MANUALLY 0x1003
72 #define CMD_REFRESH 0x1004
76 { "help", no_argument, 0, CMD_HELP },
77 { "version", no_argument, 0, CMD_VERSION },
78 { "destination", required_argument, 0, CMD_DESTINATION },
79 { "insecure", no_argument, 0, CMD_INSECURE },
80 { "sledgehammer", no_argument, 0, CMD_SLEDGE },
81 { "manually", no_argument, 0, CMD_MANUALLY },
82 { "refresh", no_argument, 0, CMD_REFRESH },
83 { "dry-run", no_argument, 0, 'n' },
84 { "verbose", no_argument, 0, 'v' },
88 // hash digest grouped by 2 digits + hash-collision counter + 2* '/' + NULL
89 typedef char HashPath[HASH_MAXBITS/4 + (HASH_MAXBITS/4/2) +
90 sizeof(unsigned int)*2 + 3];
92 struct HashDirConfiguration
94 hashFunction const *method;
95 enum { hshALL=0, hshSTART = 1, hshMIDDLE=2,
96 hshEND = 4, hshINVALID = -1 } blocks;
103 struct MatchList dst_list;
104 struct HashDirConfiguration hash_conf;
105 HashDirCollection hash_dirs;
106 size_t hash_dirs_max_size;
108 hashFunctionContext hash_context;
111 int wrapper_exit_code = 1;
112 struct Arguments const *global_args;
113 static struct SkipReason skip_reason;
115 struct WalkdownInfo global_info = {
116 .hash_conf = { .method = 0,
118 .blocksize = 0x10000 }
121 #include "vhashify-init.hc"
123 int Global_getVerbosity() {
124 return global_args->verbosity;
127 int Global_doRenew() {
131 int Global_isVserverRunning() {
133 return global_args->insecure<2;
137 showHelp(char const *cmd)
139 WRITE_MSG(1, "Usage:\n ");
142 " [-nv] [--refresh] <vserver>\n or\n ");
145 " --manually [-nv] [--] <hashdir> <path> <excludelist>\n\n"
146 " --manually ... hashify generic paths; excludelists must be generated\n"
148 " --refresh ... hashify already hashified files also\n"
149 " -n ... do not modify anything; just show what there will be\n"
150 " done (in combination with '-v')\n"
151 " -v ... verbose mode\n"
152 "Please report bugs to " PACKAGE_BUGREPORT "\n");
161 "vhashify " VERSION " -- hashifies vservers and/or directories\n"
162 "This program is part of " PACKAGE_STRING "\n\n"
163 "Copyright (C) 2005 Enrico Scholz\n"
164 VERSION_COPYRIGHT_DISCLAIMER);
169 HashDirInfo_compareDevice(void const *lhs_v, void const *rhs_v)
171 struct HashDirInfo const * const lhs = lhs_v;
172 dev_t const * const rhs = rhs_v;
174 assert(lhs!=0 && rhs!=0);
175 return lhs->device - *rhs;
179 HashDirInfo_findDevice(HashDirCollection const *coll, dev_t dev)
181 struct HashDirInfo const *res;
183 res = Vector_searchSelfOrg_const(coll, &dev,
184 HashDirInfo_compareDevice, vecSHIFT_ONCE);
186 if (res!=0) return &res->path;
190 #include "vserver-visitdir.hc"
193 checkFstat(PathInfo const * const basename,
194 struct stat * const st)
196 assert(basename->d[0] != '/');
198 // local file does not exist... strange
200 skip_reason.r = rsFSTAT;
201 if (lstat(basename->d, st)==-1) return false;
203 // this is a directory and succeeds everytime
204 if (S_ISDIR(st->st_mode))
208 skip_reason.r = rsSYMLINK;
209 if (S_ISLNK(st->st_mode)) return false;
211 // ignore special files
212 skip_reason.r = rsSPECIAL;
213 if (!S_ISREG(st->st_mode) &&
214 !S_ISDIR(st->st_mode)) return false;
216 // ignore small files
217 skip_reason.r = rsTOOSMALL;
218 if (st->st_size < HASH_MINSIZE) return false;
220 switch (Unify_isIUnlinkable(basename->d)) {
221 case unifyUNSUPPORTED : skip_reason.r = rsUNSUPPORTED; return false;
223 // do an implicit refresh on busy files when there are no active links
224 if (st->st_nlink>1 && !global_args->do_refresh) {
226 skip_reason.r = rsUNIFIED;
236 static sigjmp_buf bus_error_restore;
237 static volatile sig_atomic_t bus_error;
240 handlerSIGBUS(int UNUSED num)
243 siglongjmp(bus_error_restore, 1);
247 convertDigest(HashPath d_path)
249 static char const HEX_DIGIT[] = "0123456789abcdef";
250 hashFunctionContext * const h_ctx = &global_info.hash_context;
251 size_t d_size = h_ctx->algo->digestsize;
253 unsigned char digest[d_size];
256 if (hashFunctionContextDigest(h_ctx, digest)==-1)
260 out+1<sizeof(HashPath)-(sizeof(unsigned int)*2 + 2) && in<d_size;
262 if ((in+254)%(in<=2 ? 1 : 256) == 0 && in>0)
264 d_path[out++] = HEX_DIGIT[digest[in] >> 4];
265 d_path[out++] = HEX_DIGIT[digest[in] & 0x0f];
267 d_path[out++] = '\0';
272 #ifndef ENSC_TESTSUITE
274 addStatHash(hashFunctionContext *h_ctx, struct stat const * const st)
276 #define DECL_ATTR(X) __typeof__(st->st_##X) X
277 #define SET_ATTR(X) .X = st->st_##X
279 struct __attribute__((__packed__)) {
299 return hashFunctionContextUpdate(h_ctx, (void *)&tmp, sizeof tmp)!=-1;
303 addStatHash(hashFunctionContext UNUSED *h_ctx, struct stat const UNUSED * const st)
310 calculateHashFromFD(int fd, HashPath d_path, struct stat const * const st)
312 hashFunctionContext * const h_ctx = &global_info.hash_context;
313 void const * volatile buf = 0;
314 loff_t volatile buf_size = 0;
315 bool volatile res = false;
318 if (hashFunctionContextReset(h_ctx)==-1 ||
319 !addStatHash(h_ctx, st))
323 if (sigsetjmp(bus_error_restore,1)==0) {
325 off_t size = st->st_size;
327 while (offset < size) {
328 buf_size = size-offset;
329 if (buf_size>HASH_BLOCKSIZE) buf_size = HASH_BLOCKSIZE;
331 if ((buf=mmap(0, buf_size, PROT_READ, MAP_SHARED, fd, offset))==0) {
332 perror("mmap(<hash>)");
337 madvise(const_cast(void *)(buf), buf_size, MADV_SEQUENTIAL); // ignore error...
339 if (hashFunctionContextUpdate(h_ctx, buf, buf_size)==-1) goto out;
341 munmap(const_cast(void *)(buf), buf_size);
345 res = convertDigest(d_path);
349 if (buf!=0) munmap(const_cast(void *)(buf), buf_size);
354 calculateHash(PathInfo const *filename, HashPath d_path, struct stat const * const st)
356 int fd = open(filename->d, O_NOFOLLOW|O_NONBLOCK|O_RDONLY|O_NOCTTY);
362 int old_errno = errno;
363 WRITE_MSG(2, "Failed to open '");
364 WRITE_STR(2, filename->d);
370 if (fstat(fd, &fst)==-1 ||
371 fst.st_dev!=st->st_dev || fst.st_ino!=st->st_ino) {
372 WRITE_MSG(2, "An unexpected event occured while stating '");
373 WRITE_STR(2, filename->d);
374 WRITE_MSG(2, "'.\n");
378 if (!calculateHashFromFD(fd, d_path, st)) {
379 WRITE_MSG(2, "Failed to calculate hash for '");
380 WRITE_STR(2, filename->d);
381 WRITE_MSG(2, "'.\n");
388 if (fd!=-1) close(fd);
392 static enum { mkdirFAIL, mkdirSUCCESS, mkdirSKIP }
393 mkdirSingle(char const *path, char *end_ptr, int good_err)
396 if (mkdir(path, 0700)!=-1 || errno==EEXIST) {
400 else if (errno==good_err) {
405 int old_errno = errno;
406 WRITE_MSG(2, "mkdir('");
415 rstrchr(char *str, char c)
417 while (*str!=c) --str;
422 mkdirRecursive(char const *path)
424 if (path[0]!='/') return false; // only absolute paths
426 char buf[strlen(path)+1];
427 char * ptr = buf + sizeof(buf) - 2;
431 while (ptr>buf && (ptr = rstrchr(ptr, '/'))!=0) {
432 switch (mkdirSingle(buf, ptr, ENOENT)) {
433 case mkdirSUCCESS : break;
434 case mkdirSKIP : --ptr; continue;
435 case mkdirFAIL : return false;
438 break; // implied by mkdirSUCCESS
444 while ((ptr=strchr(ptr, '/'))!=0) {
445 switch (mkdirSingle(buf, ptr, 0)) {
447 case mkdirFAIL : return false;
448 case mkdirSUCCESS : ++ptr; continue;
456 resolveCollisions(char *result, PathInfo const *root, HashPath d_path,
457 struct stat *st, struct stat *hash_st)
459 strcpy(result, root->d); // 'root' ends on '/' already (see initHashList())
460 strcat(result, d_path);
462 char *ptr = result + strlen(result);
463 unsigned int idx = 0;
464 char buf[sizeof(int)*2 + 1];
468 ptr[sizeof(int)*2+1] = '\0';
471 len = utilvserver_fmt_xuint(buf, idx);
472 memset(ptr+1, '0', sizeof(int)*2 - len);
473 memcpy(ptr+1 + sizeof(int)*2 - len, buf, len);
475 if (lstat(result, hash_st)==-1) {
476 if (global_args->dry_run && errno!=ENOENT) {
477 int old_errno = errno;
478 WRITE_MSG(2, "lstat('");
485 else if (Unify_isUnified(st, hash_st)) {
486 skip_reason.r = rsUNIFIED;
489 else if (!Unify_isUnifyable(st, hash_st))
490 continue; // continue with next number*****
492 break; // ok, we finish here
494 if (!global_args->dry_run) {
496 if (!mkdirRecursive(result))
500 int fd = open(result, O_NOFOLLOW|O_EXCL|O_CREAT|O_WRONLY, 0200);
503 int old_errno = errno;
504 WRITE_MSG(2, "open('");
514 // HACK: avoid an additional lstat on the resulting hash-file
515 hash_st->st_size = 0;
523 checkDirEntry(PathInfo const *path, PathInfo const *basename,
525 struct stat *st, struct stat *hash_st,
528 //printf("checkDirEntry(%s, %s, %u)\n", path->d, d_path, is_dir);
530 struct WalkdownInfo const * const info = &global_info;
532 // Check if it is in the exclude/include list of the destination vserver and
533 // abort when it is not matching an allowed entry
534 skip_reason.r = rsEXCL;
535 if (MatchList_compare(&info->dst_list, path->d)!=stINCLUDE) return 0;
537 if (checkFstat(basename, st)) {
538 PathInfo const *hash_root_path;
541 *is_dir = S_ISDIR(st->st_mode);
544 !((skip_reason.r = rsWRONGDEV,
545 (hash_root_path = HashDirInfo_findDevice(&info->hash_dirs, st->st_dev))!=0) &&
546 (skip_reason.r = rsGENERAL,
547 calculateHash(basename, d_path, st)) &&
548 resolveCollisions(result_buf, hash_root_path, d_path, st, hash_st)))
561 switch (skip_reason.r) {
562 case rsDOTFILE : WRITE_MSG(1, "dotfile"); break;
563 case rsEXCL : WRITE_MSG(1, "excluded"); break;
564 case rsTOOSMALL : WRITE_MSG(1, "too small"); break;
565 case rsUNSUPPORTED : WRITE_MSG(1, "operation not supported"); break;
566 case rsFSTAT : WRITE_MSG(1, "fstat error"); break;
567 case rsSYMLINK : WRITE_MSG(1, "symlink"); break;
568 case rsUNIFIED : WRITE_MSG(1, "already unified"); break;
569 case rsSPECIAL : WRITE_MSG(1, "non regular file"); break;
570 case rsWRONGDEV : WRITE_MSG(1, "no matching device"); break;
571 case rsGENERAL : WRITE_MSG(1, "general error"); break;
572 default : assert(false); abort();
578 doit(char const *src, char const *dst,
579 struct stat const *src_st, struct stat const *dst_st,
580 PathInfo const *path)
582 if (global_args->dry_run || Global_getVerbosity()>=2) {
583 WRITE_MSG(1, "unifying '");
584 Vwrite(1, path->d, path->l);
587 if (Global_getVerbosity()>=4) {
588 WRITE_MSG(1, " (to '");
596 // abort here in dry-run mode
597 if (global_args->dry_run) return true;
599 if (dst_st->st_size==0) {
600 // file was not unified yet
602 if (Global_isVserverRunning()) {
604 if (Unify_copy (src, src_st, dst) &&
605 // the mixed 'dst' and 'src_st' params are intentionally...
606 Unify_unify(dst, src_st, src, false))
609 else if (Unify_unify(src, src_st, dst, true))
612 (void)unlink(dst); // cleanup in error-case
614 // there exists already a reference-file
615 else if (Unify_unify(dst, dst_st, src, false))
622 visitDirEntry(struct dirent const *ent)
625 char const * dirname = ent->d_name;
626 PathInfo path = global_info.state;
627 PathInfo tmp_path = {
631 char path_buf[ENSC_PI_APPSZ(path, tmp_path)];
632 char const *match = 0;
635 PathInfo_append(&path, &tmp_path, path_buf);
637 bool is_dotfile = isDotfile(dirname);
639 struct stat src_stat = { .st_mode=0 };
640 struct stat hash_stat;
641 char tmpbuf[global_info.hash_dirs_max_size +
642 sizeof(HashPath) + 2];
644 skip_reason.r = rsDOTFILE;
647 (match=checkDirEntry(&path, &tmp_path,
648 &is_dir, &src_stat, &hash_stat,
651 bool is_link = !is_dotfile && S_ISLNK(src_stat.st_mode);
653 if (Global_getVerbosity()>=1 &&
654 (Global_getVerbosity()>=3 || skip_reason.r!=rsUNIFIED) &&
655 ((!is_dotfile && !is_link) ||
656 (Global_getVerbosity()>=6 && is_dotfile) ||
657 (Global_getVerbosity()>=6 && is_link)) ) {
658 WRITE_MSG(1, " skipping '");
659 Vwrite(1, path.d, path.l);
661 if (Global_getVerbosity()>=2) printSkipReason();
669 res = visitDir(dirname, &src_stat);
671 else if (doit(dirname, match, &src_stat, &hash_stat, &path))
682 int main(int argc, char *argv[])
684 struct Arguments args = {
693 Vector_init(&global_info.hash_dirs, sizeof(struct HashDirInfo));
697 int c = getopt_long(argc, argv, "+nv",
702 case CMD_HELP : showHelp(argv[0]);
703 case CMD_VERSION : showVersion();
704 case CMD_DESTINATION : args.hash_dir = optarg; break;
705 case CMD_MANUALLY : args.mode = mdMANUALLY; break;
706 case CMD_INSECURE : args.insecure = 1; break;
707 case CMD_SLEDGE : args.insecure = 2; break;
708 case CMD_REFRESH : args.do_refresh = true; break;
709 case 'n' : args.dry_run = true; break;
710 case 'v' : ++args.verbosity; break;
712 WRITE_MSG(2, "Try '");
713 WRITE_STR(2, argv[0]);
714 WRITE_MSG(2, " --help\" for more information.\n");
721 WRITE_MSG(2, "No directory/vserver given\n");
725 if (args.hash_dir==0 && args.mode==mdMANUALLY) {
726 WRITE_MSG(2, "'--manually' requires '--destination'\n");
731 case mdMANUALLY : initModeManually(&args, argc-optind, argv+optind); break;
732 case mdVSERVER : initModeVserver (&args, argc-optind, argv+optind); break;
733 default : assert(false); return EXIT_FAILURE;
736 if (hashFunctionContextInit(&global_info.hash_context,
737 global_info.hash_conf.method)==-1) {
738 WRITE_MSG(2, "Failed to initialize hash-context\n");
742 if (Global_getVerbosity()>=1)
743 WRITE_MSG(1, "Starting to traverse directories...\n");
745 signal(SIGBUS, handlerSIGBUS);
747 Echdir(global_info.dst_list.root.d);
751 MatchList_destroy(&global_info.dst_list);
752 freeHashList(&global_info.hash_dirs);
753 hashFunctionContextFree(&global_info.hash_context);