|
| 1 | +#!./luajit |
| 2 | + |
| 3 | +local lfs |
| 4 | +local util |
| 5 | +local kotasync |
| 6 | + |
| 7 | +local function naturalsize(size) |
| 8 | + local chunk, unit = 1, 'B ' |
| 9 | + if size >= 1000*1000*1000 then |
| 10 | + chunk, unit = 1000*1000*1000, 'GB' |
| 11 | + elseif size >= 1000*1000 then |
| 12 | + chunk, unit = 1000*1000, 'MB' |
| 13 | + elseif size >= 1000 then |
| 14 | + chunk, unit = 1000, 'KB' |
| 15 | + end |
| 16 | + local fmt = chunk > 1 and "%.1f" or "%u" |
| 17 | + return string.format(fmt.." %s", size / chunk, unit) |
| 18 | +end |
| 19 | + |
| 20 | +local function make(tar_xz_path, kotasync_path, tar_xz_manifest, older_tar_xz_or_kotasync_path) |
| 21 | + local tar_xz = kotasync.TarXz:new():open(tar_xz_path, tar_xz_manifest) |
| 22 | + if older_tar_xz_or_kotasync_path then |
| 23 | + tar_xz:reorder(older_tar_xz_or_kotasync_path) |
| 24 | + end |
| 25 | + local files = {} |
| 26 | + local manifest_by_path = tar_xz.by_path |
| 27 | + if tar_xz_manifest then |
| 28 | + manifest_by_path = {} |
| 29 | + for __, f in ipairs(tar_xz.manifest) do |
| 30 | + assert(not manifest_by_path[f]) |
| 31 | + manifest_by_path[f] = true |
| 32 | + end |
| 33 | + end |
| 34 | + for e in tar_xz:each() do |
| 35 | + -- Ignore directories. |
| 36 | + if e.size ~= 0 and manifest_by_path[e.path] then |
| 37 | + table.insert(files, e) |
| 38 | + end |
| 39 | + end |
| 40 | + if tar_xz_manifest and #files ~= #tar_xz.manifest then |
| 41 | + error("mismatched manifest / archive contents") |
| 42 | + end |
| 43 | + local manifest = { |
| 44 | + filename = tar_xz_path:match("([^/]+)$"), |
| 45 | + files = files, |
| 46 | + xz_check = tonumber(tar_xz.header_stream_flags.check), |
| 47 | + } |
| 48 | + tar_xz:close() |
| 49 | + if not kotasync_path then |
| 50 | + assert(tar_xz_path:match("[.]tar.xz$")) |
| 51 | + kotasync_path = tar_xz_path:sub(1, -7).."kotasync" |
| 52 | + end |
| 53 | + kotasync.save_manifest(kotasync_path, manifest) |
| 54 | +end |
| 55 | + |
| 56 | +local function sync(state_dir, manifest_url, seed) |
| 57 | + local updater = kotasync.Updater:new(state_dir) |
| 58 | + if seed and lfs.attributes(seed, "mode") == "file" then |
| 59 | + -- If the seed is a kotasync file, we need to load it |
| 60 | + -- now, as it may get overwritten by `fetch_manifest`. |
| 61 | + local by_path = {} |
| 62 | + for i, e in ipairs(kotasync.load_manifest(seed).files) do |
| 63 | + by_path[e.path] = e |
| 64 | + end |
| 65 | + seed = by_path |
| 66 | + end |
| 67 | + updater:fetch_manifest(manifest_url) |
| 68 | + local total_files = #updater.manifest.files |
| 69 | + local last_update = 0 |
| 70 | + local delay = false --190000 |
| 71 | + local update_frequency = 0.2 |
| 72 | + local stats = updater:prepare_update(seed, function(count) |
| 73 | + local new_update = util.getTimestamp() |
| 74 | + if count ~= total_files and new_update - last_update < update_frequency then |
| 75 | + return true |
| 76 | + end |
| 77 | + last_update = new_update |
| 78 | + io.stderr:write(string.format("\ranalyzing: %4u/%4u", count, total_files)) |
| 79 | + if delay then |
| 80 | + util.usleep(delay) |
| 81 | + end |
| 82 | + return true |
| 83 | + end) |
| 84 | + io.stderr:write(string.format("\r%99s\r", "")) |
| 85 | + assert(total_files == stats.total_files) |
| 86 | + if stats.missing_files == 0 then |
| 87 | + print('nothing to update!') |
| 88 | + return |
| 89 | + end |
| 90 | + print(string.format("missing : %u/%u files", stats.missing_files, total_files)) |
| 91 | + print(string.format("reusing : %7s (%10u)", naturalsize(stats.reused_size), stats.reused_size)) |
| 92 | + print(string.format("fetching: %7s (%10u)", naturalsize(stats.download_size), stats.download_size)) |
| 93 | + io.stdout:flush() |
| 94 | + local pbar_indicators = {" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"} |
| 95 | + local pbar_size = 16 |
| 96 | + local pbar_chunk = (stats.download_size + pbar_size - 1) / pbar_size |
| 97 | + local prev_path = "" |
| 98 | + local old_progress |
| 99 | + last_update = 0 |
| 100 | + local ok, err = pcall(updater.download_update, updater, function(size, count, path) |
| 101 | + local new_update = util.getTimestamp() |
| 102 | + if size ~= stats.download_size and new_update - last_update < update_frequency then |
| 103 | + return true |
| 104 | + end |
| 105 | + last_update = new_update |
| 106 | + local padding = math.max(#prev_path, #path) |
| 107 | + local progress = math.floor(size / pbar_chunk) |
| 108 | + local pbar = pbar_indicators[#pbar_indicators]:rep(progress)..pbar_indicators[1 + math.floor(size % pbar_chunk * #pbar_indicators / pbar_chunk)]..(" "):rep(pbar_size - progress - 1) |
| 109 | + local new_progress = string.format("\rdownloading: %8s %4u/%4u %s %-"..padding.."s", size, count, stats.missing_files, pbar, path) |
| 110 | + if new_progress ~= old_progress then |
| 111 | + old_progress = new_progress |
| 112 | + io.stderr:write(new_progress) |
| 113 | + end |
| 114 | + prev_path = path |
| 115 | + if delay then |
| 116 | + util.usleep(delay) |
| 117 | + end |
| 118 | + return true |
| 119 | + end) |
| 120 | + io.stderr:write(string.format("\r%99s\r", "")) |
| 121 | + if not ok then |
| 122 | + io.stderr:write(string.format("ERROR: %s", err)) |
| 123 | + return 1 |
| 124 | + end |
| 125 | +end |
| 126 | + |
| 127 | +local help = [[ |
| 128 | +USAGE: kotasync make [-h] [--manifest TAR_XZ_MANIFEST] [--reorder OLDER_TAR_XZ_OR_KOTASYNC_FILE] TAR_XZ_FILE [KOTASYNC_FILE] |
| 129 | + kotasync sync [-h] STATE_DIR KOTASYNC_URL [SEED_DIR_OR_KOTASYNC_FILE] |
| 130 | +
|
| 131 | +options: |
| 132 | + -h, --help show this help message and exit |
| 133 | +
|
| 134 | +MAKE: |
| 135 | +
|
| 136 | + TAR_XZ_FILE source tar.xz file |
| 137 | + KOTASYNC_FILE destination kotasync file |
| 138 | +
|
| 139 | + -m, --manifest TAR_XZ_MANIFEST |
| 140 | + archive entry to use as base for manifest |
| 141 | +
|
| 142 | + -r, --reorder OLDER_TAR_XZ_OR_KOTASYNC_FILE |
| 143 | + will repack the new tar.xz with this order: |
| 144 | + ┌────────────────────┬──────────────────┬─────────────┐ |
| 145 | + │ new/modified files │ unmodified files │ folders │ |
| 146 | + │ (new order) │ (old order) │ (new order) │ |
| 147 | + └────────────────────┴──────────────────┴─────────────┘ |
| 148 | +SYNC: |
| 149 | +
|
| 150 | + STATE_DIR destination for the kotasync and update files |
| 151 | + KOTASYNC_URL URL of kotasync file |
| 152 | + SEED_DIR_OR_KOTASYNC_FILE |
| 153 | + optional seed directory / kotasync file |
| 154 | +]] |
| 155 | + |
| 156 | +local function main() |
| 157 | + local command |
| 158 | + local options = {} |
| 159 | + local arguments = {} |
| 160 | + while #arg > 0 do |
| 161 | + local a = table.remove(arg, 1) |
| 162 | + -- print(i, a) |
| 163 | + if a:match("^-(.+)$") then |
| 164 | + -- print('option', a) |
| 165 | + if a == "-h" or a == "--help" then |
| 166 | + io.stdout:write(help) |
| 167 | + return |
| 168 | + elseif command == "make" and (a == "-m" or a == "--manifest") then |
| 169 | + if #arg == 0 then |
| 170 | + io.stderr:write(string.format("ERROR: option --manifest: expected one argument\n")) |
| 171 | + return 2 |
| 172 | + end |
| 173 | + options.manifest = table.remove(arg, 1) |
| 174 | + elseif command == "make" and (a == "-r" or a == "--reorder") then |
| 175 | + if #arg == 0 then |
| 176 | + io.stderr:write(string.format("ERROR: option --reorder: expected one argument\n")) |
| 177 | + return 2 |
| 178 | + end |
| 179 | + options.reorder = table.remove(arg, 1) |
| 180 | + else |
| 181 | + io.stderr:write(string.format("ERROR: unrecognized option: %s\n", a)) |
| 182 | + return 2 |
| 183 | + end |
| 184 | + elseif command then |
| 185 | + table.insert(arguments, a) |
| 186 | + else |
| 187 | + command = a |
| 188 | + end |
| 189 | + end |
| 190 | + local fn |
| 191 | + if command == "make" then |
| 192 | + if #arguments < 1 then |
| 193 | + io.stderr:write("ERROR: not enough arguments\n") |
| 194 | + return 2 |
| 195 | + end |
| 196 | + if #arguments > 2 then |
| 197 | + io.stderr:write("ERROR: too many arguments\n") |
| 198 | + return 2 |
| 199 | + end |
| 200 | + fn = function() make(arguments[1], arguments[2], options.manifest, options.reorder) end |
| 201 | + elseif command == "sync" then |
| 202 | + if #arguments < 2 then |
| 203 | + io.stderr:write("ERROR: not enough arguments\n") |
| 204 | + return 2 |
| 205 | + end |
| 206 | + if #arguments > 3 then |
| 207 | + io.stderr:write("ERROR: too many arguments\n") |
| 208 | + return 2 |
| 209 | + end |
| 210 | + fn = function() sync(arguments[1], arguments[2], arguments[3]) end |
| 211 | + elseif not command then |
| 212 | + io.stderr:write(help) |
| 213 | + return 2 |
| 214 | + else |
| 215 | + io.stderr:write(string.format("ERROR: unrecognized command: %s\n", command)) |
| 216 | + return 2 |
| 217 | + end |
| 218 | + require("ffi/loadlib") |
| 219 | + lfs = require("libs/libkoreader-lfs") |
| 220 | + util = require("ffi/util") |
| 221 | + kotasync = require("ffi/kotasync") |
| 222 | + local ok, err = xpcall(fn, debug.traceback) |
| 223 | + if not ok then |
| 224 | + io.stderr:write(string.format("ERROR: %s\n", err)) |
| 225 | + return 3 |
| 226 | + end |
| 227 | +end |
| 228 | + |
| 229 | +os.exit(main()) |
0 commit comments