xxhash impl prototype

Author: Cryptkeeper

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "dep/jemalloc"]
 	path = dep/jemalloc
 	url = https://github.com/jemalloc/jemalloc
+[submodule "dep/xxHash"]
+	path = dep/xxHash
+	url = https://github.com/Cyan4973/xxHash

CMakeLists.txt

@@ -9,6 +9,11 @@ set(ENABLE_CJSON_TEST OFF)
 set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 add_subdirectory(dep/cJSON)
 
+# link xxHash statically
+set(XXHASH_BUILD_XXHSUM OFF)
+option(BUILD_SHARED_LIBS OFF)
+add_subdirectory(dep/xxHash/build/cmake xxhash_build EXCLUDE_FROM_ALL)
+
 # use strict C11
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_C_STANDARD_REQUIRED ON)
@@ -29,16 +34,16 @@ add_executable(fsautoproc ${SOURCES} ${HEADERS})
 add_compile_definitions(JEMALLOC_NO_DEMANGLE)
 
 target_link_directories(fsautoproc PRIVATE dep dep/jemalloc/lib)
-target_link_libraries(fsautoproc PRIVATE cjson pthread libjemalloc.a)
+target_link_libraries(fsautoproc PRIVATE cjson pthread libjemalloc.a xxhash)
 target_include_directories(fsautoproc PRIVATE include dep dep/jemalloc/include)
 
 install(TARGETS fsautoproc DESTINATION bin)
 
 # libdeng shared library for unit tests
-add_library(deng STATIC src/deng.c src/index.c src/fs.c src/je.c)
+add_library(deng STATIC src/deng.c src/index.c src/fs.c src/je.c src/xx.c)
 
 target_link_directories(deng PUBLIC dep/jemalloc/lib)
-target_link_libraries(deng PUBLIC libjemalloc.a)
+target_link_libraries(deng PUBLIC libjemalloc.a xxhash)
 target_include_directories(deng PUBLIC include dep dep/jemalloc/include)
 
 # unit tests

dep/xxHash

@@ -14,6 +14,7 @@ struct inode_s {
   char* fp;            ///< File path (string duplicated)
   uint64_t fphash;     ///< File path hash value
   struct fsstat_s st;  ///< File stat info structure
+  uint64_t xx;         ///< xxHash64 hash value
   struct inode_s* next;///< Next node in the index map
 };
 
@@ -78,10 +79,11 @@ int indexread(struct index_s* idx, FILE* s);
 /// @param fp The file path to use for the new node, duplicated internally
 /// @param fphash The file path hash value to use for the new node
 /// @param st The file stat info to use for the new node
+/// @param xx The xxHash64 hash value to use for the new node
 /// @return The pointer to the new node in the index map, otherwise NULL is
 /// returned and `errno` is set.
 struct inode_s* indexput(struct index_s* idx, const char* fp, uint64_t fphash,
-                         const struct fsstat_s* st);
+                         const struct fsstat_s* st, uint64_t xx);
 
 /// @brief Frees all nodes in the index map.
 /// @param idx The index to free

include/index.h

@@ -0,0 +1,34 @@
+/// @file xx.h
+/// @brief xxHash64 file hashing functions.
+#ifndef FSAUTOPROC_XX_H
+#define FSAUTOPROC_XX_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/// @def XXBUFSZE
+/// @brief Size of the buffer used for reading files to hash.
+#define XXBUFSZE 65536
+
+/// @struct xxreq_s
+/// @brief File hash request structure for processing file hash changes.
+struct xxreq_s {
+  uint8_t* b;                ///< Buffer for reading file data
+  const char* fp;            ///< Filepath to hash
+  const struct fsstat_s* pst;///< Previous file stat info, may be NULL
+  const struct fsstat_s* cst;///< Current file stat info
+  uint64_t pxx;              ///< Previous xxHash64 value, or 0 if unknown
+};
+
+/// @brief Processes a file hash request. If the file's last modified time or
+/// size differ from the previous stat info \p pst, the file is re-hashed
+/// and the current hash value is returned. Otherwise the previous \p pxx hash
+/// value is returned indicating no change.
+/// @param req The file hash request structure containing the filepath, previous
+/// and current stat info, and previous hash value (if known).
+/// @param cause Optional string indicating the cause for the hash request, for
+/// logging purposes. May be NULL.
+/// @return 0 if an error occurred, otherwise the current hash value.
+uint64_t xxupdate(struct xxreq_s* req, const char* cause);
+
+#endif//FSAUTOPROC_XX_H

include/xx.h

@@ -10,6 +10,13 @@
 #include "fs.h"
 #include "index.h"
 #include "log.h"
+#include "xx.h"
+
+/// @def XXBUFSZE
+/// @brief Size of the buffer used for reading files to hash.
+#define XXBUFSZE 65536
+
+static uint8_t xxbuf[XXBUFSZE];///< Shared buffer for reading files to hash
 
 /// @struct deng_state_s
 /// @brief Search state context provided to the diff engine as user data which
@@ -58,14 +65,15 @@ static int stagepre(const char* fp, const struct fsstat_s* st, void* udata) {
   // attempt to match file in previous index
   struct inode_s* prev = indexfind(mach->lastmap, fp, fphash);
 
-  // lookup from previous iteration or insert new record and lookup
-  struct inode_s* curr = indexfind(mach->thismap, fp, fphash);
-  if (curr == NULL)
-    if ((curr = indexput(mach->thismap, fp, fphash, st)) == NULL) return -1;
+  // insert node for current index
+  struct inode_s* curr = indexput(mach->thismap, fp, fphash, st, 0);
+  if (!curr) return -1;
 
   if (prev != NULL) {
-    if (prev->st.lmod != st->lmod || prev->st.fsze != st->fsze)
-      callevent(mach, DENG_FEVENT_MOD, curr);
+    // file existed in previous index, check if modified
+    struct xxreq_s req = {xxbuf, fp, &prev->st, st, prev->xx};
+    curr->xx = xxupdate(&req, "stage=pre,node=existing");
+    if (req.pxx != curr->xx) callevent(mach, DENG_FEVENT_MOD, curr);
   } else {
     callevent(mach, DENG_FEVENT_NEW, curr);
   }
@@ -86,16 +94,28 @@ static int stagepost(const char* fp, const struct fsstat_s* st, void* udata) {
 
   if (mach->ffn != NULL && mach->ffn(fp)) return 0;// skip filtered files
 
-  const uint64_t fphash = indexhash(fp);
+  struct xxreq_s req = {xxbuf, fp, NULL, st, 0};
+  char* cause = NULL;
 
+  const uint64_t fphash = indexhash(fp);
   struct inode_s* curr = indexfind(mach->thismap, fp, fphash);
   if (curr != NULL) {
-    curr->st = *st;// update the file info in the current index
-    return 0;
+    req.pst = &curr->st;
+    req.pxx = curr->xx;
+    cause = "stage=post,node=existing";
+  } else {
+    cause = "stage=post,node=new";
   }
 
-  if ((curr = indexput(mach->thismap, fp, fphash, st)) == NULL) return -1;
-  callevent(mach, DENG_FEVENT_NEW, curr);
+  const uint64_t xx = xxupdate(&req, cause);
+
+  if (curr != NULL) {
+    curr->xx = xx;// update hash of existing record
+  } else {
+    // file was not found in previous stage, insert new record
+    if ((curr = indexput(mach->thismap, fp, fphash, st, xx)) == NULL) return -1;
+    callevent(mach, DENG_FEVENT_NEW, curr);
+  }
 
   return 0;
 }

src/deng.c

@@ -64,8 +64,9 @@ int indexwrite(struct index_s* idx, FILE* s) {
   int err = 0;
   for (long i = 0; i < idx->size; i++) {
     struct inode_s* node = fl[i];
-    const int n = snprintf(lbuf, sizeof(lbuf), "%s,%" PRIu64 ",%" PRIu64 "\n",
-                           node->fp, node->st.lmod, node->st.fsze);
+    const int n = snprintf(lbuf, sizeof(lbuf),
+                           "%s,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "\n", node->fp,
+                           node->st.lmod, node->st.fsze, node->xx);
     if (fwrite(lbuf, n, 1, s) != 1) {
       err = -1;
       break;
@@ -78,10 +79,11 @@ int indexwrite(struct index_s* idx, FILE* s) {
 int indexread(struct index_s* idx, FILE* s) {
   char fp[INDEXMAXFP] = {0}; /* fscanf filepath string buffer */
   struct fsstat_s st = {0};  /* fscanf file stat structure */
-  while (fscanf(s, "%[^,],%" PRIu64 ",%" PRIu64 "\n", fp, &st.lmod, &st.fsze) ==
-         3) {
+  uint64_t xx = 0;           /* fscanf xxHash64 value */
+  while (fscanf(s, "%[^,],%" PRIu64 ",%" PRIu64 ",%" PRIu64 "\n", fp, &st.lmod,
+                &st.fsze, &xx) == 4) {
     const uint64_t fphash = indexhash(fp);
-    if (indexput(idx, fp, fphash, &st) == NULL) return -1;
+    if (indexput(idx, fp, fphash, &st, xx) == NULL) return -1;
   }
   return 0;
 }
@@ -101,14 +103,15 @@ static void indexappend(struct ibucket_s* bucket, struct inode_s* node) {
 }
 
 struct inode_s* indexput(struct index_s* idx, const char* fp,
-                         const uint64_t fphash, const struct fsstat_s* st) {
+                         const uint64_t fphash, const struct fsstat_s* st,
+                         const uint64_t xx) {
   struct inode_s* node = je_malloc(sizeof(struct inode_s));
   if (node == NULL) return NULL;
   if ((fp = je_strdup(fp)) == NULL) {// duplicate filepath string
     je_free(node);
     return NULL;
   }
-  *node = (struct inode_s) {(char*) fp, fphash, *st, NULL};
+  *node = (struct inode_s) {(char*) fp, fphash, *st, xx, NULL};
   struct ibucket_s* bucket = &idx->buckets[indexbucket(node->fphash)];
   indexappend(bucket, node);
   idx->size++;

src/index.c

@@ -252,7 +252,7 @@ static bool filterjunk(const char* fp) {
     if (initargs.verbose) log_info("[j] %s", fp);
   } else {
     const struct fsstat_s st = {0};
-    indexput(&goodmap, fp, fphash, &st);// mark as known good
+    indexput(&goodmap, fp, fphash, &st, 0);// mark as known good
   }
   return junk;
 }

src/main.c

@@ -17,17 +17,19 @@
 #include "lcmd.h"
 #include "log.h"
 #include "tm.h"
+#include "xx.h"
 
 /// @struct thrd_s
 /// @brief Initialized worker thread in the thread pool.
 struct thrd_s {
-  _Atomic bool initd;  ///< Thread initialized flag
-  _Atomic bool rsrvd;  ///< Work request reservation flag
-  _Atomic bool canwork;///< Work request ready to process flag
-  struct tpreq_s work; ///< Work request to process
-  pthread_t tid;       ///< System thread identifier
-  _Bool fdsopen;       ///< File descriptor set open flag
-  struct fdset_s fds;  ///< Output file descriptor set
+  _Atomic bool initd;     ///< Thread initialized flag
+  _Atomic bool rsrvd;     ///< Work request reservation flag
+  _Atomic bool canwork;   ///< Work request ready to process flag
+  struct tpreq_s work;    ///< Work request to process
+  pthread_t tid;          ///< System thread identifier
+  _Bool fdsopen;          ///< File descriptor set open flag
+  struct fdset_s fds;     ///< Output file descriptor set
+  uint8_t xxbuf[XXBUFSZE];///< Buffer for reading files to hash
 };
 
 static struct thrd_s** thrds; ///< Thread pool worker threads array
@@ -57,8 +59,16 @@ static void* tpentrypoint(void* arg) {
     if ((err = lcmdexec(req->cs, req->node->fp, self->fds, req->flags)))
       log_error("thread execution error: %d", err);
     if (req->flags & (LCTRIG_NEW | LCTRIG_MOD)) {
-      if ((err = fsstat(req->node->fp, &req->node->st)))
+      struct fsstat_s st = {0};
+      if ((err = fsstat(req->node->fp, &st))) {
         log_error("stat error: %d", err);
+      } else {
+        // check if the stats have changed, if so, re-hash
+        struct xxreq_s xx = {self->xxbuf, req->node->fp, &req->node->st, &st,
+                             req->node->xx};
+        req->node->xx = xxupdate(&xx, "stage=tp");
+        req->node->st = st;// update node with potentially new stat info
+      }
     }
 
     atomic_store(&self->rsrvd, false);// release the reservation

src/tp.c

@@ -0,0 +1,60 @@
+/// @file xx.c
+/// @brief xxHash64 file hashing function implementation.
+#include "xx.h"
+
+#include <assert.h>
+#include <stdio.h>
+
+#include "xxHash/xxhash.h"
+
+#include "fs.h"
+#include "log.h"
+
+/// @brief Computes the xxHash64 hash of the data read from the given file
+/// using a buffer for reading. The file is read in chunks until EOF.
+/// @param b Buffer for reading file data. Must not be NULL.
+/// @param f FILE object to read data from. Must not be NULL. The file should be
+/// opened in binary mode and positioned at the start of the data.
+/// @return Hash value of the file data, or 0 if an error occurred.
+static uint64_t xxhash(uint8_t* b, FILE* f) {
+  assert(b != NULL);
+  assert(f != NULL);
+  XXH3_state_t* state = XXH3_createState();
+  if (!state) return 0;
+  if (XXH3_64bits_reset(state)) goto doerr;
+  size_t n;
+  while ((n = fread(b, 1, XXBUFSZE, f)) > 0) {
+    if (XXH3_64bits_update(state, b, n)) goto doerr;
+  }
+  uint64_t h = XXH3_64bits_digest(state);
+  XXH3_freeState(state);
+  return h;
+doerr:
+  XXH3_freeState(state);
+  return 0;
+}
+
+/// @brief Computes the xxHash64 hash of the file located at the given filepath.
+/// The file is opened, read in chunks until EOF, and then closed.
+/// @param b Buffer for reading file data. Must not be NULL.
+/// @param fp The filepath of the file to hash. Must not be NULL.
+/// @return The hash value of the file, or 0 if an error occurred (e.g. file
+/// could not be opened).
+static uint64_t xxhashfp(uint8_t* b, const char* fp) {
+  assert(b != NULL);
+  assert(fp != NULL);
+  FILE* f = fopen(fp, "rb");
+  if (!f) return 0;
+  const uint64_t h = xxhash(b, f);
+  fclose(f);
+  return h;
+}
+
+uint64_t xxupdate(struct xxreq_s* req, const char* cause) {
+  assert(req != NULL);
+  const struct fsstat_s* pst = req->pst;
+  if (pst != NULL && pst->lmod == req->cst->lmod && pst->fsze == req->cst->fsze)
+    return req->pxx;// no change
+  if (cause != NULL) log_info("hashing `%s` (%s)", req->fp, cause);
+  return xxhashfp(req->b, req->fp);
+}