Refactoring: Use rhash_update_fd() to hash files

Author: rhash

ChangeLog

@@ -1,3 +1,6 @@
+Sat 19 Apr 2025 Aleksey Kravchenko
+	* Speed up file reading while hashing a file
+
 Tue 15 Apr 2025 Aleksey Kravchenko
 	* Bugfix: Fix processing of ./ directory
 	* Support --blake3 option for the BLAKE3 hash function

Makefile

@@ -157,7 +157,7 @@ $(RHASH_BINARY): $(OBJECTS) $(LIBRHASH_PATH)
 # NOTE: dependences were generated by 'gcc -Ilibrhash -MM *.c'
 # we are using plain old makefile style to support BSD make
 calc_sums.o: calc_sums.c calc_sums.h common_func.h file.h hash_check.h \
- file_set.h hash_print.h output.h parse_cmdline.h rhash_main.h \
+ file_set.h hash_print.h output.h parse_cmdline.h platform.h rhash_main.h \
  win_utils.h librhash/rhash.h librhash/rhash_torrent.h
 	$(CC) -c $(CFLAGS) $< -o $@
 

calc_sums.c

@@ -4,6 +4,7 @@
 #include "hash_print.h"
 #include "output.h"
 #include "parse_cmdline.h"
+#include "platform.h"
 #include "rhash_main.h"
 #include "win_utils.h"
 #include "librhash/rhash.h"
@@ -207,15 +208,15 @@ static void re_init_rhash_context(struct file_info* info)
  */
 int calc_sums(struct file_info* info)
 {
-	FILE* fd = 0;
+	int fd = -1;
 	int res;
 
 	assert(info->file);
 	if (FILE_ISSTDIN(info->file)) {
-		fd = stdin;
+		fd = 0;
 #ifdef _WIN32
 		/* using 0 instead of _fileno(stdin). _fileno() is undefined under 'gcc -ansi' */
-		if (setmode(0, _O_BINARY) < 0)
+		if (setmode(fd, _O_BINARY) < 0)
 			return -1;
 #endif
 	} else {
@@ -230,9 +231,9 @@ int calc_sums(struct file_info* info)
 			return 0;
 
 		if (!FILE_ISDATA(info->file)) {
-			fd = file_fopen(info->file, FOpenRead | FOpenBin);
+			fd = file_open(info->file, FOpenReadBin);
 			/* quietly skip unreadble files */
-			if (!fd)
+			if (fd < 0)
 				return -1;
 		}
 	}
@@ -248,7 +249,7 @@ int calc_sums(struct file_info* info)
 		if (percents_output->update != 0) {
 			rhash_set_callback(info->rctx, (rhash_callback_t)percents_output->update, info);
 		}
-		res = rhash_file_update(info->rctx, fd);
+		res = rhash_update_fd(info->rctx, fd, RHASH_MAX_FILE_SIZE);
 	}
 	if (res != -1 && !opt.bt_batch_file)
 		rhash_final(info->rctx, 0); /* finalize hashing */
@@ -257,8 +258,8 @@ int calc_sums(struct file_info* info)
 	info->size = info->rctx->msg_size - info->msg_offset;
 	rhash_data.total_size += info->size;
 
-	if (fd && !FILE_ISSTDIN(info->file))
-		fclose(fd);
+	if (fd >= 0 && !FILE_ISSTDIN(info->file))
+		close(fd);
 	return res;
 }
 

file.c

@@ -291,7 +291,7 @@ static int file_statw(file_t* file);
  * @param dir_path (nullable) directory path to prepend to printable path
  * @param print_path printable path, which encoding shall be detected
  * @param init_flags bit flags, helping to detect the encoding
- * @return encoding on success, -1 on fail with error code stored in errno
+ * @return encoding on success, -1 on failure with error code stored in errno
  */
 static int detect_path_encoding(file_t* file, wchar_t* dir_path, const char* print_path, unsigned init_flags)
 {
@@ -354,7 +354,7 @@ static int detect_path_encoding(file_t* file, wchar_t* dir_path, const char* pri
  * @param prepend_dir the directory to prepend to the print_path, to construct the file path, can be NULL
  * @param print_path the printable representation of the file path
  * @param init_flags initialization flags
- * @return 0 on success, -1 on fail with error code stored in errno
+ * @return 0 on success, -1 on failure with error code stored in errno
  */
 int file_init_by_print_path(file_t* file, file_t* prepend_dir, const char* print_path, unsigned init_flags)
 {
@@ -617,7 +617,7 @@ void file_swap(file_t* first, file_t* second)
  * @param str the string to insert into/append to the source file path
  * @param operation the operation determinating how to modify the file path, can be one of the values
  *                  FModifyAppendSuffix, FModifyInsertBeforeExtension, FModifyRemoveExtension, FModifyGetParentDir
- * @return allocated and modified file path on success, NULL on fail
+ * @return allocated and modified file path on success, NULL on failure
  */
 static char* get_modified_path(const char* path, const char* str, int operation)
 {
@@ -654,7 +654,7 @@ static char* get_modified_path(const char* path, const char* str, int operation)
  * @param str the string to insert into/append to the source file path
  * @param operation the operation determinating how to modify the file path, can be one of the values
  *                  FModifyAppendSuffix, FModifyInsertBeforeExtension, FModifyRemoveExtension, FModifyGetParentDir
- * @return allocated and modified file path on success, NULL on fail
+ * @return allocated and modified file path on success, NULL on failure
  */
 static tpath_t get_modified_tpath(ctpath_t path, const char* str, int operation)
 {
@@ -694,7 +694,7 @@ static tpath_t get_modified_tpath(ctpath_t path, const char* str, int operation)
  * @param str the string to insert into/append to the source file path
  * @param operation the operation to do on src file, can be one of the values
  *                  FModifyAppendSuffix, FModifyInsertBeforeExtension, FModifyRemoveExtension, FModifyGetParentDir
- * @return 0 on success, -1 on fail with error code stored in errno
+ * @return 0 on success, -1 on failure with error code stored in errno
  */
 int file_modify_path(file_t* dst, file_t* src, const char* str, int operation)
 {
@@ -729,7 +729,7 @@ int file_modify_path(file_t* dst, file_t* src, const char* str, int operation)
  * Retrieve file information (type, size, mtime) into file_t fields.
  *
  * @param file the file information
- * @return 0 on success, -1 on fail with error code stored in errno
+ * @return 0 on success, -1 on failure with error code stored in errno
  */
 static int file_statw(file_t* file)
 {
@@ -777,7 +777,7 @@ static int file_statw(file_t* file)
  *
  * @param file the file information
  * @param fstat_flags bitmask consisting of FileStatModes bits
- * @return 0 on success, -1 on fail with error code stored in errno
+ * @return 0 on success, -1 on failure with error code stored in errno
  */
 int file_stat(file_t* file, int fstat_flags)
 {
@@ -822,11 +822,65 @@ int file_stat(file_t* file, int fstat_flags)
 }
 
 /**
- * Open the file and return its decriptor.
+ * Open the file and return POSIX file descriptor.
  *
- * @param file the file information, including the path
- * @param fopen_flags bitmask consisting of FileFOpenModes bits
- * @return file descriptor on success, NULL on fail with error code stored in errno
+ * On Windows, the file is always opened in binary mode (`_O_BINARY`)
+ * and with `_SH_DENYNO` (shared read/write access).
+ * In read-only mode (`FOpenRead`), the file is opened with a hint for sequential access:
+ * - Windows: Uses `_O_SEQUENTIAL`.
+ * - POSIX: Uses `posix_fadvise(POSIX_FADV_SEQUENTIAL)` if available.
+ *
+ * @param file file information, including the path (must not be NULL)
+ * @param open_flags bitmask of `FOpenRead`, `FOpenWrite`
+ * @return POSIX file descriptor on success, -1 on failure with error code stored in errno
+ */
+int file_open(file_t* file, int open_flags)
+{
+	const int possible_oflags[4] = {
+#if defined(_WIN32)
+		0, _O_RDONLY | _O_BINARY | _O_SEQUENTIAL,
+		_O_WRONLY | _O_BINARY, _O_RDWR | _O_BINARY
+#else
+# if !defined(O_BINARY)
+#  define O_BINARY 0
+# endif
+		0, O_RDONLY | O_BINARY, O_WRONLY | O_BINARY, O_RDWR | O_BINARY
+#endif
+	};
+	const int oflags = possible_oflags[open_flags & FOpenRW];
+	int fd;
+	assert((open_flags & FOpenRW) != 0);
+	if (!file->real_path) {
+		errno = EINVAL;
+		return -1;
+	}
+#ifdef _WIN32
+	{
+		fd = _wsopen(file->real_path, oflags, _SH_DENYNO, 0);
+		if (fd < 0 && errno == EINVAL)
+			errno = ENOENT;
+		return fd;
+	}
+#else
+	fd = open(file->real_path, oflags, 0);
+# if _POSIX_C_SOURCE >= 200112L && defined(POSIX_FADV_SEQUENTIAL)
+	if (fd >= 0)
+		posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+# endif /* _POSIX_C_SOURCE >= 200112L && defined(POSIX_FADV_SEQUENTIAL) */
+	return fd;
+#endif
+}
+
+/**
+ * Open the file and return a standard C file stream pointer.
+ *
+ * On Windows, the file is opened with shared access (`_SH_DENYNO`)
+ * In read-only mode (`FOpenRead`), a hint for sequential access is applied.
+ *
+ * @param file the file information, including the path (must not be NULL)
+ * @param fopen_flags bitmask of `FOpenRead`, `FOpenWrite`, and optionally `FOpenBin`.
+ *                    Must include at least one of `FOpenRead` or `FOpenWrite`.
+ * @return valid `FILE*` stream on success, NULL on failure with error code stored in errno
  */
 FILE* file_fopen(file_t* file, int fopen_flags)
 {
@@ -868,7 +922,7 @@ FILE* file_fopen(file_t* file, int fopen_flags)
  *
  * @param from the source file
  * @param to the destination path
- * @return 0 on success, -1 on fail with error code stored in errno
+ * @return 0 on success, -1 on failure with error code stored in errno
  */
 int file_rename(const file_t* from, const file_t* to)
 {
@@ -889,7 +943,7 @@ int file_rename(const file_t* from, const file_t* to)
  * Rename a given file to *.bak, if it exists.
  *
  * @param file the file to move
- * @return 0 on success, -1 on fail with error code stored in errno
+ * @return 0 on success, -1 on failure with error code stored in errno
  */
 int file_move_to_bak(file_t* file)
 {
@@ -962,7 +1016,7 @@ int file_is_readable(file_t* file)
  *
  * @param list the file_list_t structure to initialize
  * @param file the file to open
- * @return 0 on success, -1 on fail with error code stored in errno
+ * @return 0 on success, -1 on failure with error code stored in errno
  */
 int file_list_open(file_list_t* list, file_t* file)
 {
@@ -1040,7 +1094,7 @@ struct WIN_DIR_t
  * Open directory iterator for reading the directory content.
  *
  * @param dir_path directory path
- * @return pointer to directory stream, NULL on fail with error code stored in errno
+ * @return pointer to directory stream, NULL on failure with error code stored in errno
  */
 WIN_DIR* win_opendir(const char* dir_path)
 {

file.h

@@ -132,6 +132,7 @@ enum FileFOpenModes {
 	FOpenWriteBin = FOpenWrite | FOpenBin,
 	FOpenRWBin    = FOpenRW | FOpenBin,
 };
+int file_open(file_t* file, int open_flags);
 FILE* file_fopen(file_t* file, int fopen_flags);
 
 int file_rename(const file_t* from, const file_t* to);