xxhash impl prototype

Author: Cryptkeeper

.github/workflows/cmake.yml

@@ -25,7 +25,7 @@ jobs:
           make -j $(nproc)
         working-directory: ${{github.workspace}}/dep/jemalloc
       - name: Configure CMake
-        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_COMPILER=${{matrix.compiler}}
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_COMPILER=${{matrix.compiler}} -DCMAKE_POLICY_VERSION_MINIMUM=3.5
       - name: Build
         run: cmake --build ${{github.workspace}}/build
       - name: CTest

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "dep/jemalloc"]
 	path = dep/jemalloc
 	url = https://github.com/jemalloc/jemalloc
+[submodule "dep/xxHash"]
+	path = dep/xxHash
+	url = https://github.com/Cyan4973/xxHash

CMakeLists.txt

@@ -9,6 +9,11 @@ set(ENABLE_CJSON_TEST OFF)
 set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 add_subdirectory(dep/cJSON)
 
+# link xxHash statically
+set(XXHASH_BUILD_XXHSUM OFF)
+option(BUILD_SHARED_LIBS OFF)
+add_subdirectory(dep/xxHash/build/cmake xxhash_build EXCLUDE_FROM_ALL)
+
 # use strict C11
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_C_STANDARD_REQUIRED ON)
@@ -29,16 +34,16 @@ add_executable(fsautoproc ${SOURCES} ${HEADERS})
 add_compile_definitions(JEMALLOC_NO_DEMANGLE)
 
 target_link_directories(fsautoproc PRIVATE dep dep/jemalloc/lib)
-target_link_libraries(fsautoproc PRIVATE cjson pthread libjemalloc.a)
+target_link_libraries(fsautoproc PRIVATE cjson pthread libjemalloc.a xxhash)
 target_include_directories(fsautoproc PRIVATE include dep dep/jemalloc/include)
 
 install(TARGETS fsautoproc DESTINATION bin)
 
 # libdeng shared library for unit tests
-add_library(deng STATIC src/deng.c src/index.c src/fs.c src/je.c)
+add_library(deng STATIC src/deng.c src/index.c src/fs.c src/je.c src/xx.c)
 
 target_link_directories(deng PUBLIC dep/jemalloc/lib)
-target_link_libraries(deng PUBLIC libjemalloc.a)
+target_link_libraries(deng PUBLIC libjemalloc.a xxhash)
 target_include_directories(deng PUBLIC include dep dep/jemalloc/include)
 
 # unit tests
@@ -47,3 +52,8 @@ enable_testing()
 add_executable(test_deng test/test_deng.c)
 target_link_libraries(test_deng PRIVATE deng)
 add_test(NAME deng COMMAND test_deng)
+
+add_executable(test_xx test/test_xx.c src/xx.c)
+target_link_libraries(test_xx PRIVATE xxhash)
+target_include_directories(test_xx PRIVATE include dep)
+add_test(NAME xx COMMAND test_xx)

README.md

@@ -35,6 +35,7 @@ C11 is used for basic atomic boolean operations when scheduling work across thre
 Git submodules provide:
 
 - [cJSON](https://github.com/DaveGamble/cJSON)
+- [xxHash](https://github.com/Cyan4973/xxHash)
 - [jemalloc](https://github.com/jemalloc/jemalloc) (requires autoconf during build)
 
 ### Usage
@@ -57,6 +58,8 @@ Options:
   -x <file>   Exclusive lock file path
 ```
 
+When first indexing a new file structure, you should use the `-u` flag to avoid applying any newly configured rules to the existing file structure (unless desired, i.e. an initial "clean up" mutation of an unmaintained directory).
+
 ### Basic Configuration
 
 `example.fsautoproc.json` provides a basic example configuration file. The configuration file is a JSON array of objects, each object representing a desired "action" at a grouping level of your choosing. Each action object has the following properties:

dep/xxHash

@@ -14,6 +14,7 @@ struct inode_s {
   char* fp;            ///< File path (string duplicated)
   uint64_t fphash;     ///< File path hash value
   struct fsstat_s st;  ///< File stat info structure
+  uint64_t xx;         ///< xxHash64 hash value
   struct inode_s* next;///< Next node in the index map
 };
 
@@ -78,10 +79,11 @@ int indexread(struct index_s* idx, FILE* s);
 /// @param fp The file path to use for the new node, duplicated internally
 /// @param fphash The file path hash value to use for the new node
 /// @param st The file stat info to use for the new node
+/// @param xx The xxHash64 hash value to use for the new node
 /// @return The pointer to the new node in the index map, otherwise NULL is
 /// returned and `errno` is set.
 struct inode_s* indexput(struct index_s* idx, const char* fp, uint64_t fphash,
-                         const struct fsstat_s* st);
+                         const struct fsstat_s* st, uint64_t xx);
 
 /// @brief Frees all nodes in the index map.
 /// @param idx The index to free

include/index.h

@@ -0,0 +1,33 @@
+/// @file xx.h
+/// @brief xxHash64 file hashing functions.
+#ifndef FSAUTOPROC_XX_H
+#define FSAUTOPROC_XX_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/// @def XXBUFSZE
+/// @brief Size of the buffer used for reading files to hash.
+#define XXBUFSZE 65536
+
+/// @struct xxreq_s
+/// @brief File hash request structure for processing file hash changes.
+struct xxreq_s {
+  uint8_t* b;                ///< Buffer for reading file data
+  const char* fp;            ///< Filepath to hash
+  const struct fsstat_s* pst;///< Previous file stat info, may be NULL
+  const struct fsstat_s* cst;///< Current file stat info
+  uint64_t pxx;              ///< Previous xxHash64 value, or 0 if unknown
+  const char* cause;         ///< Optional cause string for logging, may be NULL
+};
+
+/// @brief Processes a file hash request. If the file's last modified time or
+/// size differ from the previous stat info \p pst, the file is re-hashed
+/// and the current hash value is returned. Otherwise the previous \p pxx hash
+/// value is returned indicating no change.
+/// @param req The file hash request structure containing the filepath, previous
+/// and current stat info, and previous hash value (if known).
+/// @return 0 if an error occurred, otherwise the current hash value.
+uint64_t xxupdate(const struct xxreq_s* req);
+
+#endif//FSAUTOPROC_XX_H

include/xx.h

@@ -10,6 +10,13 @@
 #include "fs.h"
 #include "index.h"
 #include "log.h"
+#include "xx.h"
+
+/// @def XXBUFSZE
+/// @brief Size of the buffer used for reading files to hash.
+#define XXBUFSZE 65536
+
+static uint8_t xxbuf[XXBUFSZE];///< Shared buffer for reading files to hash
 
 /// @struct deng_state_s
 /// @brief Search state context provided to the diff engine as user data which
@@ -58,15 +65,25 @@ static int stagepre(const char* fp, const struct fsstat_s* st, void* udata) {
   // attempt to match file in previous index
   struct inode_s* prev = indexfind(mach->lastmap, fp, fphash);
 
-  // lookup from previous iteration or insert new record and lookup
-  struct inode_s* curr = indexfind(mach->thismap, fp, fphash);
-  if (curr == NULL)
-    if ((curr = indexput(mach->thismap, fp, fphash, st)) == NULL) return -1;
+  // insert node for current index
+  struct inode_s* curr = indexput(mach->thismap, fp, fphash, st, 0);
+  if (!curr) return -1;
+
+  struct xxreq_s xx = {0};
+  xx.b = xxbuf;
+  xx.fp = fp;
+  xx.cst = st;
 
   if (prev != NULL) {
-    if (prev->st.lmod != st->lmod || prev->st.fsze != st->fsze)
-      callevent(mach, DENG_FEVENT_MOD, curr);
+    // file existed in previous index, check if modified
+    xx.pst = &prev->st;
+    xx.pxx = prev->xx;
+    xx.cause = "stage=pre,node=existing";
+    curr->xx = xxupdate(&xx);
+    if (xx.pxx != curr->xx) callevent(mach, DENG_FEVENT_MOD, curr);
   } else {
+    xx.cause = "stage=pre,node=new";
+    curr->xx = xxupdate(&xx);// hash new record
     callevent(mach, DENG_FEVENT_NEW, curr);
   }
 
@@ -86,16 +103,27 @@ static int stagepost(const char* fp, const struct fsstat_s* st, void* udata) {
 
   if (mach->ffn != NULL && mach->ffn(fp)) return 0;// skip filtered files
 
-  const uint64_t fphash = indexhash(fp);
+  struct xxreq_s xx = {0};
+  xx.b = xxbuf;
+  xx.fp = fp;
+  xx.cst = st;
 
+  const uint64_t fphash = indexhash(fp);
   struct inode_s* curr = indexfind(mach->thismap, fp, fphash);
   if (curr != NULL) {
-    curr->st = *st;// update the file info in the current index
-    return 0;
-  }
+    xx.pst = &curr->st;
+    xx.pxx = curr->xx;
+    xx.cause = "stage=post,node=existing";
+    curr->xx = xxupdate(&xx);// update hash of existing record
+  } else {
+    xx.cause = "stage=post,node=new";
+    const uint64_t cxx = xxupdate(&xx);// hash new record
 
-  if ((curr = indexput(mach->thismap, fp, fphash, st)) == NULL) return -1;
-  callevent(mach, DENG_FEVENT_NEW, curr);
+    // file was not found in previous stage, insert new record
+    if ((curr = indexput(mach->thismap, fp, fphash, st, cxx)) == NULL)
+      return -1;
+    callevent(mach, DENG_FEVENT_NEW, curr);
+  }
 
   return 0;
 }
@@ -151,6 +179,7 @@ int dengsearch(const char* sd, deng_filter_t filter,
   if ((err = execstage(&mach, sd, stagepre))) goto ret;
   if ((err = checkremoved(&mach))) goto ret;
   if ((err = execstage(&mach, sd, stagepost))) goto ret;
+  if ((err = checkremoved(&mach))) goto ret;
 ret:
   return err;
 }

src/deng.c

@@ -64,8 +64,9 @@ int indexwrite(struct index_s* idx, FILE* s) {
   int err = 0;
   for (long i = 0; i < idx->size; i++) {
     struct inode_s* node = fl[i];
-    const int n = snprintf(lbuf, sizeof(lbuf), "%s,%" PRIu64 ",%" PRIu64 "\n",
-                           node->fp, node->st.lmod, node->st.fsze);
+    const int n = snprintf(lbuf, sizeof(lbuf),
+                           "%s,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "\n", node->fp,
+                           node->st.lmod, node->st.fsze, node->xx);
     if (fwrite(lbuf, n, 1, s) != 1) {
       err = -1;
       break;
@@ -78,10 +79,11 @@ int indexwrite(struct index_s* idx, FILE* s) {
 int indexread(struct index_s* idx, FILE* s) {
   char fp[INDEXMAXFP] = {0}; /* fscanf filepath string buffer */
   struct fsstat_s st = {0};  /* fscanf file stat structure */
-  while (fscanf(s, "%[^,],%" PRIu64 ",%" PRIu64 "\n", fp, &st.lmod, &st.fsze) ==
-         3) {
+  uint64_t xx = 0;           /* fscanf xxHash64 value */
+  while (fscanf(s, "%[^,],%" PRIu64 ",%" PRIu64 ",%" PRIu64 "\n", fp, &st.lmod,
+                &st.fsze, &xx) == 4) {
     const uint64_t fphash = indexhash(fp);
-    if (indexput(idx, fp, fphash, &st) == NULL) return -1;
+    if (indexput(idx, fp, fphash, &st, xx) == NULL) return -1;
   }
   return 0;
 }
@@ -101,14 +103,15 @@ static void indexappend(struct ibucket_s* bucket, struct inode_s* node) {
 }
 
 struct inode_s* indexput(struct index_s* idx, const char* fp,
-                         const uint64_t fphash, const struct fsstat_s* st) {
+                         const uint64_t fphash, const struct fsstat_s* st,
+                         const uint64_t xx) {
   struct inode_s* node = je_malloc(sizeof(struct inode_s));
   if (node == NULL) return NULL;
   if ((fp = je_strdup(fp)) == NULL) {// duplicate filepath string
     je_free(node);
     return NULL;
   }
-  *node = (struct inode_s) {(char*) fp, fphash, *st, NULL};
+  *node = (struct inode_s) {(char*) fp, fphash, *st, xx, NULL};
   struct ibucket_s* bucket = &idx->buckets[indexbucket(node->fphash)];
   indexappend(bucket, node);
   idx->size++;

src/index.c

@@ -252,7 +252,7 @@ static bool filterjunk(const char* fp) {
     if (initargs.verbose) log_info("[j] %s", fp);
   } else {
     const struct fsstat_s st = {0};
-    indexput(&goodmap, fp, fphash, &st);// mark as known good
+    indexput(&goodmap, fp, fphash, &st, 0);// mark as known good
   }
   return junk;
 }