diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5eea484..30c0590 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,8 @@ jobs: run: uv pip install -e ".[dev]" - name: Run ruff run: uv run ruff check . + - name: Run ruff format check + run: uv run ruff format --check . - name: Run mypy run: uv run mypy jsoncsv diff --git a/README.md b/README.md index 5d4d005..f2ca629 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Just use them. ## Requirements -- Python 3.8 or higher +- Python 3.11 or higher ## Quick Start diff --git a/jsoncsv/dumptool.py b/jsoncsv/dumptool.py index 7ce0d98..6bfe2ce 100644 --- a/jsoncsv/dumptool.py +++ b/jsoncsv/dumptool.py @@ -67,8 +67,8 @@ def load_headers( class DumpExcel(Dump, ReadHeadersMixin): def initialize(self, **kwargs: Any) -> None: super().initialize(**kwargs) - self._read_row = kwargs.get('read_row') - self._sort_type = kwargs.get('sort_type') + self._read_row = kwargs.get("read_row") + self._sort_type = kwargs.get("sort_type") def prepare(self) -> None: headers, datas = self.load_headers(self.fin, self._read_row, self._sort_type) @@ -104,10 +104,7 @@ def write_headers(self) -> None: self.csv_writer.writeheader() def write_obj(self, obj: dict[str, JsonType]) -> None: - patched_obj: dict[str, str] = { - key: self.patch_value(value) - for key, value in obj.items() - } + patched_obj: dict[str, str] = {key: self.patch_value(value) for key, value in obj.items()} assert self.csv_writer is not None self.csv_writer.writerow(patched_obj) @@ -121,8 +118,8 @@ class DumpXLS(DumpExcel): def initialize(self, **kwargs: Any) -> None: super().initialize(**kwargs) - self.sheet = kwargs.get('sheet', 'Sheet1') - self.wb = xlwt.Workbook(encoding='utf-8') + self.sheet = kwargs.get("sheet", "Sheet1") + self.wb = xlwt.Workbook(encoding="utf-8") self.ws = self.wb.add_sheet(self.sheet) self.row = 0 self.cloumn = 0 diff --git a/jsoncsv/jsontool.py b/jsoncsv/jsontool.py index 3d99636..09efcc3 100644 --- a/jsoncsv/jsontool.py +++ b/jsoncsv/jsontool.py @@ -19,9 +19,9 @@ ) __all__ = [ - 'convert_json', - 'expand', - 'restore', + "convert_json", + "expand", + "restore", ] # Type alias for the func parameter in convert_json @@ -90,7 +90,7 @@ def from_leaf(leafs: Iterable[LeafInputType]) -> JsonType: return dict(child) # type: ignore[arg-type] -def expand(origin: JsonType, separator: str = '.', safe: bool = False) -> dict[str, JsonType]: +def expand(origin: JsonType, separator: str = ".", safe: bool = False) -> dict[str, JsonType]: root = origin leafs = gen_leaf(root) @@ -105,7 +105,7 @@ def expand(origin: JsonType, separator: str = '.', safe: bool = False) -> dict[s return expobj -def restore(expobj: dict[str, JsonType], separator: str = '.', safe: bool = False) -> JsonType: +def restore(expobj: dict[str, JsonType], separator: str = ".", safe: bool = False) -> JsonType: leafs: list[tuple[DecodedPathType, JsonType]] = [] items = expobj.items() @@ -113,7 +113,7 @@ def restore(expobj: dict[str, JsonType], separator: str = '.', safe: bool = Fals for key, value in items: path: DecodedPathType = decode_safe_key(key, separator) if safe else key.split(separator) - if key == '': + if key == "": path = [] leafs.append((path, value)) @@ -126,13 +126,13 @@ def convert_json( fin: io.TextIOBase, fout: io.TextIOBase, func: ConvertFunc, - separator: str = '.', + separator: str = ".", safe: bool = False, json_array: bool = False, ) -> None: - ''' + """ ensure fin/fout is TextIO - ''' + """ if func not in [expand, restore]: raise ValueError("unknow convert_json type") @@ -158,4 +158,4 @@ def gen_objs_from_array() -> Iterator[JsonType]: new = func(obj, separator=separator, safe=safe) content = json.dumps(new, ensure_ascii=False) fout.write(content) - fout.write('\n') + fout.write("\n") diff --git a/jsoncsv/main.py b/jsoncsv/main.py index 16c22fa..11ee365 100644 --- a/jsoncsv/main.py +++ b/jsoncsv/main.py @@ -15,40 +15,22 @@ def separator_type(sep: str) -> str: if len(sep) != 1: - raise click.BadOptionUsage(option_name='separator', - message='separator can only be a char') + raise click.BadOptionUsage(option_name="separator", message="separator can only be a char") if sep == unit_char: - raise click.BadOptionUsage(option_name='separator', - message='separator can not be `\\` ') + raise click.BadOptionUsage(option_name="separator", message="separator can not be `\\` ") return sep @click.command() -@click.option('-A', - '--array', - 'json_array', - is_flag=True, - default=False, - help='read input file as json array') -@click.option('-s', - '--sep', - 'separator', - type=separator_type, - default='.', - help='separator') -@click.option('--safe', is_flag=True, help='use safe mode') -@click.option('-r', - '--restore', - 'restore', - is_flag=True, - help='restore expanded json') -@click.option('-e', - '--expand', - 'expand', - is_flag=True, - help='expand json (default True)') -@click.argument('input', type=click.File('r', encoding='utf-8'), default='-') -@click.argument('output', type=click.File('w', encoding='utf-8'), default='-') +@click.option( + "-A", "--array", "json_array", is_flag=True, default=False, help="read input file as json array" +) +@click.option("-s", "--sep", "separator", type=separator_type, default=".", help="separator") +@click.option("--safe", is_flag=True, help="use safe mode") +@click.option("-r", "--restore", "restore", is_flag=True, help="restore expanded json") +@click.option("-e", "--expand", "expand", is_flag=True, help="expand json (default True)") +@click.argument("input", type=click.File("r", encoding="utf-8"), default="-") +@click.argument("output", type=click.File("w", encoding="utf-8"), default="-") def jsoncsv( output: io.TextIOBase, input: io.TextIOBase, @@ -59,42 +41,34 @@ def jsoncsv( json_array: bool, ) -> None: if expand and restore: - raise click.UsageError('can not choose both, default is `-e`') + raise click.UsageError("can not choose both, default is `-e`") func: Callable[..., Any] func = expand_fn if not restore else restore_fn - convert_json(input, - output, - func, - separator=separator, - safe=safe, - json_array=json_array) + convert_json(input, output, func, separator=separator, safe=safe, json_array=json_array) input.close() output.close() @click.command() -@click.option('-t', - '--type', - 'type_', - type=click.Choice(['csv', 'xls']), - default='csv', - help='choose dump format') -@click.option('-r', - '--row', - type=int, - default=None, - help='number of pre-read `row` lines to load `headers`') -@click.option('-s', - '--sort', - 'sort_', - is_flag=True, - default=False, - help='enable sort the headers keys') -@click.argument('input', type=click.File('r', encoding='utf-8'), default='-') -@click.argument('output', type=click.Path(), default='-') +@click.option( + "-t", + "--type", + "type_", + type=click.Choice(["csv", "xls"]), + default="csv", + help="choose dump format", +) +@click.option( + "-r", "--row", type=int, default=None, help="number of pre-read `row` lines to load `headers`" +) +@click.option( + "-s", "--sort", "sort_", is_flag=True, default=False, help="enable sort the headers keys" +) +@click.argument("input", type=click.File("r", encoding="utf-8"), default="-") +@click.argument("output", type=click.Path(), default="-") def mkexcel( output: str, input: io.TextIOBase, @@ -107,13 +81,13 @@ def mkexcel( klass = dumptool.DumpXLS # Open file in appropriate mode based on type - if output == '-': - fout: Any = sys.stdout.buffer if type_ == 'xls' else sys.stdout + if output == "-": + fout: Any = sys.stdout.buffer if type_ == "xls" else sys.stdout dump_excel(input, fout, klass, read_row=row, sort_type=sort_) else: - mode = 'wb' if type_ == 'xls' else 'w' - encoding = None if type_ == 'xls' else 'utf-8' - newline = '' if type_ == 'csv' else None + mode = "wb" if type_ == "xls" else "w" + encoding = None if type_ == "xls" else "utf-8" + newline = "" if type_ == "csv" else None with open(output, mode, encoding=encoding, newline=newline) as fout: dump_excel(input, fout, klass, read_row=row, sort_type=sort_) diff --git a/jsoncsv/utils.py b/jsoncsv/utils.py index 68fa200..8af33f6 100644 --- a/jsoncsv/utils.py +++ b/jsoncsv/utils.py @@ -2,14 +2,14 @@ # 2016.11.20 # Type aliases for JSON data structures -JsonType = dict[str, 'JsonType'] | list['JsonType'] | str | int | float | bool | None +JsonType = dict[str, "JsonType"] | list["JsonType"] | str | int | float | bool | None PathType = list[int | str] # Can contain ints (array indices) or strings (dict keys) DecodedPathType = list[str] # Decoded paths from keys are always strings LeafType = tuple[PathType, JsonType] # Type for leafs that can contain either PathType (from gen_leaf) or DecodedPathType (from restore) LeafInputType = LeafType | tuple[DecodedPathType, JsonType] -unit_char = '\\' +unit_char = "\\" def encode_safe_key(path: list[str], separator: str) -> str: @@ -20,13 +20,13 @@ def encode_safe_key(path: list[str], separator: str) -> str: def decode_safe_key(key: str, separator: str) -> list[str]: path: list[str] = [] - p = '' + p = "" escape = False for char in key: if escape and char == separator: path.append(p) - p = '' + p = "" escape = False elif escape and char == unit_char: p += unit_char @@ -36,6 +36,6 @@ def decode_safe_key(key: str, separator: str) -> list[str]: else: p += char - if p != '': + if p != "": path.append(p) return path diff --git a/tests/test_dumptool.py b/tests/test_dumptool.py index a92cf09..e8c8961 100644 --- a/tests/test_dumptool.py +++ b/tests/test_dumptool.py @@ -7,39 +7,54 @@ class TestDumpTool(unittest.TestCase): - # FIXME (使用虚拟文件) def test_dumpexcel_csv(self): - with open('./fixture/files/expand.1.json', encoding='utf-8') as fin, \ - open('./fixture/files/tmp.output.1.csv', 'w', encoding='utf-8', newline='') as fout: + with ( + open("./fixture/files/expand.1.json", encoding="utf-8") as fin, + open("./fixture/files/tmp.output.1.csv", "w", encoding="utf-8", newline="") as fout, + ): dump_excel(fin, fout, DumpCSV) - with open('./fixture/files/output.1.csv', encoding='utf-8') as output, \ - open('./fixture/files/tmp.output.1.csv', encoding='utf-8') as fout: + with ( + open("./fixture/files/output.1.csv", encoding="utf-8") as output, + open("./fixture/files/tmp.output.1.csv", encoding="utf-8") as fout, + ): self.assertEqual(output.read(), fout.read()) def test_dumpexcel_csv_with_sort(self): - with open('./fixture/files/expand.1.json', encoding='utf-8') as fin, \ - open('./fixture/files/tmp.output.1.sort.csv', 'w', encoding='utf-8', newline='') as fout: + with ( + open("./fixture/files/expand.1.json", encoding="utf-8") as fin, + open( + "./fixture/files/tmp.output.1.sort.csv", "w", encoding="utf-8", newline="" + ) as fout, + ): dump_excel(fin, fout, DumpCSV, sort_type=True) - with open('./fixture/files/output.1.sort.csv', encoding='utf-8') as output, \ - open('./fixture/files/tmp.output.1.sort.csv', encoding='utf-8') as fout: + with ( + open("./fixture/files/output.1.sort.csv", encoding="utf-8") as output, + open("./fixture/files/tmp.output.1.sort.csv", encoding="utf-8") as fout, + ): self.assertEqual(output.read(), fout.read()) def test_dumpcexcel_xls(self): - with open('./fixture/files/expand.1.json', encoding='utf-8') as fin, \ - open('./fixture/files/tmp.output.1.xls', 'wb') as fout: + with ( + open("./fixture/files/expand.1.json", encoding="utf-8") as fin, + open("./fixture/files/tmp.output.1.xls", "wb") as fout, + ): dump_excel(fin, fout, DumpXLS) def test_dump_csv_with_non_ascii(self): - with open('./fixture/files/expand.2.json', encoding='utf-8') as fin, \ - open('./fixture/files/tmp.output.2.csv', 'w', encoding='utf-8', newline='') as fout: + with ( + open("./fixture/files/expand.2.json", encoding="utf-8") as fin, + open("./fixture/files/tmp.output.2.csv", "w", encoding="utf-8", newline="") as fout, + ): dump_excel(fin, fout, DumpCSV) def test_dump_xls_with_non_ascii(self): - with open('./fixture/files/expand.2.json', encoding='utf-8') as fin, \ - open('./fixture/files/tmp.output.2.xls', 'wb') as fout: + with ( + open("./fixture/files/expand.2.json", encoding="utf-8") as fin, + open("./fixture/files/tmp.output.2.xls", "wb") as fout, + ): dump_excel(fin, fout, DumpXLS) def test_dump_xls_with_dict(self): diff --git a/tests/test_escape.py b/tests/test_escape.py index 5e3d7e7..dd9f963 100644 --- a/tests/test_escape.py +++ b/tests/test_escape.py @@ -7,29 +7,28 @@ class Testescape(unittest.TestCase): - def test_all(self): - path = ['A', 'B', '..', '\\.\\ww'] + path = ["A", "B", "..", "\\.\\ww"] - for sep in 'AB.w': + for sep in "AB.w": key = encode_safe_key(path, sep) _path = decode_safe_key(key, sep) self.assertListEqual(path, _path) def test_encode(self): - path = ['A', 'B', 'C', 'www.xxx.com'] - sep = '.' + path = ["A", "B", "C", "www.xxx.com"] + sep = "." key = encode_safe_key(path, sep) - self.assertEqual(key, 'A\\.B\\.C\\.www.xxx.com') + self.assertEqual(key, "A\\.B\\.C\\.www.xxx.com") def test_decode(self): - key = 'A\\.B\\.C\\.www.xxx.com' - sep = '.' + key = "A\\.B\\.C\\.www.xxx.com" + sep = "." path = decode_safe_key(key, sep) - self.assertEqual(path[0], 'A') - self.assertEqual(path[1], 'B') - self.assertEqual(path[2], 'C') - self.assertEqual(path[3], 'www.xxx.com') + self.assertEqual(path[0], "A") + self.assertEqual(path[1], "B") + self.assertEqual(path[2], "C") + self.assertEqual(path[3], "www.xxx.com") diff --git a/tests/test_jsoncsv.py b/tests/test_jsoncsv.py index 4637b92..7cf82c0 100644 --- a/tests/test_jsoncsv.py +++ b/tests/test_jsoncsv.py @@ -11,54 +11,50 @@ class Testjsoncsv(unittest.TestCase): def test_jsoncsv_expand(self): runner = CliRunner() - args = ['-e', 'fixture/files/raw.0.json', - 'fixture/files/tmp.expand.0.json'] + args = ["-e", "fixture/files/raw.0.json", "fixture/files/tmp.expand.0.json"] result = runner.invoke(jsoncsv, args=args) assert result.exit_code == 0 def test_jsoncsv_expand_with_json_array(self): runner = CliRunner() - args = ['-e', 'fixture/files/raw.1.json', - 'fixture/files/tmp.expand.1.json', '-A'] + args = ["-e", "fixture/files/raw.1.json", "fixture/files/tmp.expand.1.json", "-A"] result = runner.invoke(jsoncsv, args=args) assert result.exit_code == 0 def test_jsoncsv_expand_restore(self): runner = CliRunner(echo_stdin=True) - result = runner.invoke(jsoncsv, - args=['-e', 'fixture/files/raw.2.json', - 'fixture/files/tmp.expand.2.json']) + result = runner.invoke( + jsoncsv, args=["-e", "fixture/files/raw.2.json", "fixture/files/tmp.expand.2.json"] + ) assert result.exit_code == 0 - result = runner.invoke(jsoncsv, - args=['-r', 'fixture/files/tmp.expand.2.json', - 'fixture/files/tmp.restore.2.json']) + result = runner.invoke( + jsoncsv, + args=["-r", "fixture/files/tmp.expand.2.json", "fixture/files/tmp.restore.2.json"], + ) assert result.exit_code == 0 - with open('fixture/files/raw.2.json') as f: + with open("fixture/files/raw.2.json") as f: input_data = [json.loads(line) for line in f] - with open('fixture/files/tmp.restore.2.json') as f: + with open("fixture/files/tmp.restore.2.json") as f: resotre_data = [json.loads(line) for line in f] self.assertEqual(input_data, resotre_data) def test_jsoncsv_with_error_args(self): runner = CliRunner() - args = ['-s', 'aa', '-e', 'fixture/files/raw.0.json', - 'fixture/files/tmp.expand.0.json'] + args = ["-s", "aa", "-e", "fixture/files/raw.0.json", "fixture/files/tmp.expand.0.json"] result = runner.invoke(jsoncsv, args=args) assert result.exit_code != 0 def test_jsoncsv_with_error_sep_args(self): runner = CliRunner() - args = ['-s', '\\', '-e', 'fixture/files/raw.0.json', - 'fixture/files/tmp.expand.0.json'] + args = ["-s", "\\", "-e", "fixture/files/raw.0.json", "fixture/files/tmp.expand.0.json"] result = runner.invoke(jsoncsv, args=args) assert result.exit_code != 0 def test_jsoncsv_with_error_args_expand_and_restore(self): runner = CliRunner() - args = ['-r', '-e', 'fixture/files/raw.0.json', - 'fixture/files/tmp.expand.0.json'] + args = ["-r", "-e", "fixture/files/raw.0.json", "fixture/files/tmp.expand.0.json"] result = runner.invoke(jsoncsv, args=args) assert result.exit_code != 0 diff --git a/tests/test_jsontool.py b/tests/test_jsontool.py index 39b16ab..141d3da 100644 --- a/tests/test_jsontool.py +++ b/tests/test_jsontool.py @@ -8,7 +8,6 @@ class TestJSONTool(unittest.TestCase): - def test_string(self): s = "sss" exp = expand(s) @@ -28,9 +27,7 @@ def test_dict(self): "w": 5, "t": { "m": 0, - "x": { - "y": "z" - }, + "x": {"y": "z"}, }, } @@ -40,13 +37,7 @@ def test_dict(self): self.assertDictEqual(s, _s) def test_complex(self): - s = [ - {"s": 0}, - {"t": ["2", {"x": "z"}]}, - 0, - "w", - ["x", "g", 1] - ] + s = [{"s": 0}, {"t": ["2", {"x": "z"}]}, 0, "w", ["x", "g", 1]] exp = expand(s) _s = restore(exp) @@ -60,16 +51,16 @@ def test_complex(self): def test_is_array_index(self): self.assertTrue(is_array_index([0, 1, 2, 3])) - self.assertTrue(is_array_index(['0', '1', '2', '3'])) + self.assertTrue(is_array_index(["0", "1", "2", "3"])) # string order - self.assertTrue(is_array_index(['0', '1', '10', '2', '3', '4', '5', '6', '7', '8', '9'])) + self.assertTrue(is_array_index(["0", "1", "10", "2", "3", "4", "5", "6", "7", "8", "9"])) self.assertFalse(is_array_index([1, 2, 3])) - self.assertFalse(is_array_index(['0', 1, 2])) + self.assertFalse(is_array_index(["0", 1, 2])) def test_unicode(self): data = [ {"河流名字": "长江", "河流长度": "6000千米"}, - {"河流名字": "黄河", "河流长度": "5000千米"} + {"河流名字": "黄河", "河流长度": "5000千米"}, ] expobj = expand(data) @@ -81,8 +72,8 @@ def test_expand_with_safe(self): "api.a.com": {"qps": 100, "p95": 20, "p99": 100}, } expobj = expand(data, safe=True) - self.assertEqual(expobj['api.a.com\\.p95'], 20) - self.assertEqual(expobj['api.a.com\\.p99'], 100) + self.assertEqual(expobj["api.a.com\\.p95"], 20) + self.assertEqual(expobj["api.a.com\\.p99"], 100) origin = restore(expobj, safe=True) self.assertEqual(origin, data) @@ -98,7 +89,6 @@ def test_expand_and_restore(self): class TestConvertJSON(unittest.TestCase): - def test_convert_expand(self): fin = io.StringIO('{"a":{"b":3}}\n{"a":{"c":4}}\n') fout = io.StringIO() diff --git a/tests/test_mkexcel.py b/tests/test_mkexcel.py index 61b0646..78e6b9d 100644 --- a/tests/test_mkexcel.py +++ b/tests/test_mkexcel.py @@ -11,21 +11,18 @@ class Testmkexcel(unittest.TestCase): def test_mkexcel_csv(self): runner = CliRunner() - args = ['fixture/files/expand.0.json', - 'fixture/files/tmp.expand.0.csv'] + args = ["fixture/files/expand.0.json", "fixture/files/tmp.expand.0.csv"] result = runner.invoke(mkexcel, args=args) assert result.exit_code == 0 def test_mkexcel_xls(self): runner = CliRunner() - args = ['-t', 'xls', 'fixture/files/expand.0.json', - 'fixture/files/tmp.expand.0.xls'] + args = ["-t", "xls", "fixture/files/expand.0.json", "fixture/files/tmp.expand.0.xls"] result = runner.invoke(mkexcel, args=args) assert result.exit_code == 0 def test_mkexcel_with_error(self): runner = CliRunner() - args = ['-t', 'xlsx', 'fixture/files/expand.0.json', - 'fixture/files/tmp.expand.0.xls'] + args = ["-t", "xlsx", "fixture/files/expand.0.json", "fixture/files/tmp.expand.0.xls"] result = runner.invoke(mkexcel, args=args) assert result.exit_code == 2