Skip to content

Commit 0f3612e

Browse files
author
hongwei.quhw
committed
indexCodec
1 parent 91bcc5a commit 0f3612e

File tree

4 files changed

+246
-8
lines changed

4 files changed

+246
-8
lines changed

‎rdf-file-core/src/main/java/com/alipay/rdf/file/codec/RowNosqlIndexCodec.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
* @Author: hongwei.quhw 2021/6/27 3:48 下午
2222
*/
2323
public class RowNosqlIndexCodec extends AbstractRowCodec {
24-
private static final String INDEX_PLACE_KEY = "rowCodecIndex";
24+
private static final String INDEX_PLACE_KEY = "rowCodecIndexPlace";
2525
// (start|end) 代表在 (第一个字段|最后一个字段)
2626
private static final String INDEX_START = "start";
2727
private static final String INDEX_END = "end";
@@ -44,8 +44,8 @@ public String serialize(RowCodecContext rccCtx) {
4444
try {
4545
ctx.codecType = RdfFileFunctionSpi.CodecType.SERIALIZE;
4646
ctx.field = bmw.getProperty(columnMeta.getName());
47-
// 对非空字段进行序列化
48-
if (null != ctx.field) {
47+
// 对非空字段进行序列化 或者 模板配置存在默认值
48+
if (null != ctx.field || RdfFileUtil.isNotBlank(columnMeta.getDefaultValue())) {
4949
ctx.columnMeta = columnMeta;
5050
ctx.fileConfig = rccCtx.fileConfig;
5151
String value = (String) rccCtx.rd.getOutput().execute(ctx);
@@ -92,11 +92,11 @@ public String preDeserialize(String line, RowCodecContext rccCtx) {
9292
String split = RdfFileUtil.getRowSplit(rccCtx.fileConfig);
9393
String colMeta;
9494
if (RdfFileUtil.isBlank(indexPlace) || INDEX_END.equalsIgnoreCase(indexPlace)) {
95-
int endIdx = line.indexOf(split);
95+
int endIdx = line.lastIndexOf(split);
9696
colMeta = line.substring(endIdx + split.length());
9797
line = line.substring(0, endIdx);
9898
} else if (INDEX_START.equalsIgnoreCase(indexPlace)) {
99-
int startIdx = line.lastIndexOf(split);
99+
int startIdx = line.indexOf(split);
100100
colMeta = line.substring(0, startIdx);
101101
line = line.substring(startIdx + split.length());
102102
} else {
@@ -127,12 +127,12 @@ public void deserialize(String line, RowCodecContext rccCtx) {
127127
String[] columnValues = rccCtx.columnValues;
128128

129129
// 反序列化索引字段
130-
String idxArray[] = (String[]) rccCtx.ext;
130+
Integer idxArray[] = (Integer[]) rccCtx.ext;
131131

132132
List<FileColumnMeta> columnMetas = new ArrayList<FileColumnMeta>(idxArray.length);
133-
for (String index : idxArray) {
133+
for (Integer index : idxArray) {
134134
for (FileColumnMeta columnMeta : rccCtx.columnMetas) {
135-
if (index.equals(String.valueOf(columnMeta.getColIndex()))) {
135+
if (index.equals(columnMeta.getColIndex())) {
136136
columnMetas.add(columnMeta);
137137
}
138138
}

‎rdf-file-core/src/test/java/com/alipay/rdf/file/codec/RowNosqlIndexTest.java‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
package com.alipay.rdf.file.codec;
2+
3+
import com.alipay.rdf.file.interfaces.FileFactory;
4+
import com.alipay.rdf.file.interfaces.FileReader;
5+
import com.alipay.rdf.file.interfaces.FileWriter;
6+
import com.alipay.rdf.file.model.FileConfig;
7+
import com.alipay.rdf.file.model.StorageConfig;
8+
import com.alipay.rdf.file.util.DateUtil;
9+
import com.alipay.rdf.file.util.TemporaryFolderUtil;
10+
import org.junit.After;
11+
import org.junit.Assert;
12+
import org.junit.Before;
13+
import org.junit.Test;
14+
15+
import java.io.*;
16+
import java.math.BigDecimal;
17+
import java.util.Date;
18+
import java.util.HashMap;
19+
import java.util.Map;
20+
21+
/**
22+
* @Author: hongwei.quhw 2021/9/10 2:14 下午
23+
*/
24+
public class RowNosqlIndexTest {
25+
TemporaryFolderUtil tf = new TemporaryFolderUtil();
26+
27+
@Before
28+
public void setUp() throws IOException {
29+
tf.create();
30+
}
31+
32+
// 参数指定
33+
@Test
34+
public void test1() throws IOException {
35+
String filePath = tf.getRoot().getAbsolutePath();
36+
FileConfig config = new FileConfig(new File(filePath, "test.txt").getAbsolutePath(),
37+
"/codec/index/template/template1.json", new StorageConfig("nas"));
38+
config.setRowCodecMode("index");
39+
40+
FileWriter fileWriter = FileFactory.createWriter(config);
41+
42+
Map<String, Object> head = new HashMap<String, Object>();
43+
head.put("totalCount", 2);
44+
head.put("totalAmount", new BigDecimal("23.22"));
45+
fileWriter.writeHead(head);
46+
47+
Map<String, Object> body = new HashMap<String, Object>();
48+
49+
Date testDate = DateUtil.parse("2017-01-03 12:22:33", "yyyy-MM-dd HH:mm:ss");
50+
51+
body.put("seq", "seq12345");
52+
body.put("instSeq", "303");
53+
body.put("gmtApply", testDate);
54+
body.put("date", testDate);
55+
body.put("dateTime", testDate);
56+
body.put("applyNumber", new BigDecimal("12"));
57+
body.put("amount", new BigDecimal("1.22"));
58+
body.put("age", new Integer(33));
59+
body.put("longN", new Long(33));
60+
body.put("bol", true);
61+
body.put("memo", "memo1");
62+
fileWriter.writeRow(body);
63+
64+
testDate = DateUtil.parse("2016-02-03 12:22:33", "yyyy-MM-dd HH:mm:ss");
65+
body = new HashMap<String, Object>();
66+
body.put("seq", "seq234567");
67+
body.put("instSeq", "505");
68+
body.put("gmtApply", testDate);
69+
body.put("date", testDate);
70+
body.put("dateTime", testDate);
71+
body.put("applyNumber", new BigDecimal("12"));
72+
//body.put("amount", new BigDecimal("1.09"));
73+
body.put("age", 66);
74+
body.put("longN", 125);
75+
body.put("bol", false);
76+
body.put("memo", "memo2");
77+
fileWriter.writeRow(body);
78+
79+
fileWriter.close();
80+
81+
//校验文件
82+
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(new File(config.getFilePath())), "UTF-8"));
83+
Assert.assertEquals("总笔数:2|总金额:23.22|colMeta(idx:0,1)", reader.readLine());
84+
Assert.assertEquals("流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注", reader.readLine());
85+
Assert.assertEquals("seq12345|303|2017-01-03 12:22:33|20170103|20170103 12:22:33|12|1.22|33|33|true|memo1|colMeta(idx:0,1,2,3,4,5,6,7,8,9,10)", reader.readLine());
86+
Assert.assertEquals("seq234567|505|2016-02-03 12:22:33|20160203|20160203 12:22:33|12|66|125|false|memo2|colMeta(idx:0,1,2,3,4,5,7,8,9,10)", reader.readLine());
87+
reader.close();
88+
89+
FileReader fileReader = FileFactory.createReader(config);
90+
head = fileReader.readHead(HashMap.class);
91+
Assert.assertEquals(2L, head.get("totalCount"));
92+
Assert.assertEquals(new BigDecimal("23.22"), head.get("totalAmount"));
93+
94+
body = fileReader.readRow(HashMap.class);
95+
body = fileReader.readRow(HashMap.class);
96+
97+
Assert.assertEquals("seq234567", body.get("seq"));
98+
Assert.assertEquals("505", body.get("instSeq"));
99+
Assert.assertEquals("2016-02-03 12:22:33", DateUtil.format((Date) body.get("gmtApply"), "yyyy-MM-dd HH:mm:ss"));
100+
Assert.assertEquals(new BigDecimal("12"), body.get("applyNumber"));
101+
Assert.assertEquals(null, body.get("amount"));
102+
Assert.assertEquals(66, body.get("age"));
103+
Assert.assertEquals(125L, body.get("longN"));
104+
Assert.assertEquals(false, body.get("bol"));
105+
Assert.assertEquals("memo2", body.get("memo"));
106+
107+
fileReader.close();
108+
}
109+
110+
// 模板指定codec， 模板配置索引位置 空但是存在默认值
111+
@Test
112+
public void test2() throws IOException {
113+
String filePath = tf.getRoot().getAbsolutePath();
114+
FileConfig config = new FileConfig(new File(filePath, "test.txt").getAbsolutePath(),
115+
"/codec/index/template/template2.json", new StorageConfig("nas"));
116+
117+
FileWriter fileWriter = FileFactory.createWriter(config);
118+
119+
Map<String, Object> head = new HashMap<String, Object>();
120+
head.put("totalCount", 2);
121+
head.put("totalAmount", new BigDecimal("23.22"));
122+
fileWriter.writeHead(head);
123+
124+
Map<String, Object> body = new HashMap<String, Object>();
125+
126+
Date testDate = DateUtil.parse("2017-01-03 12:22:33", "yyyy-MM-dd HH:mm:ss");
127+
128+
body.put("seq", "seq12345");
129+
body.put("instSeq", "303");
130+
body.put("gmtApply", testDate);
131+
body.put("date", testDate);
132+
body.put("dateTime", testDate);
133+
body.put("applyNumber", new BigDecimal("12"));
134+
body.put("amount", new BigDecimal("1.22"));
135+
body.put("age", new Integer(33));
136+
body.put("longN", new Long(33));
137+
body.put("bol", true);
138+
//body.put("memo", "memo1");
139+
fileWriter.writeRow(body);
140+
141+
testDate = DateUtil.parse("2016-02-03 12:22:33", "yyyy-MM-dd HH:mm:ss");
142+
body = new HashMap<String, Object>();
143+
body.put("seq", "");
144+
body.put("instSeq", "505");
145+
body.put("gmtApply", testDate);
146+
body.put("date", testDate);
147+
body.put("dateTime", testDate);
148+
body.put("applyNumber", new BigDecimal("12"));
149+
//body.put("amount", new BigDecimal("1.09"));
150+
body.put("age", 66);
151+
body.put("longN", 125);
152+
body.put("bol", false);
153+
body.put("memo", "memo2");
154+
fileWriter.writeRow(body);
155+
156+
fileWriter.close();
157+
158+
//校验文件
159+
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(new File(config.getFilePath())), "UTF-8"));
160+
Assert.assertEquals("colMeta(idx:0,1)|总笔数:2|总金额:23.22", reader.readLine());
161+
Assert.assertEquals("流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注", reader.readLine());
162+
Assert.assertEquals("colMeta(idx:0,1,2,3,4,5,6,7,8,9,10)|seq12345|303|2017-01-03 12:22:33|20170103|20170103 12:22:33|12|1.22|33|33|true|defaultMemo", reader.readLine());
163+
Assert.assertEquals("colMeta(idx:0,1,2,3,4,5,7,8,9,10)||505|2016-02-03 12:22:33|20160203|20160203 12:22:33|12|66|125|false|memo2", reader.readLine());
164+
reader.close();
165+
166+
FileReader fileReader = FileFactory.createReader(config);
167+
head = fileReader.readHead(HashMap.class);
168+
Assert.assertEquals(2L, head.get("totalCount"));
169+
Assert.assertEquals(new BigDecimal("23.22"), head.get("totalAmount"));
170+
171+
body = fileReader.readRow(HashMap.class);
172+
Assert.assertEquals("defaultMemo", body.get("memo"));
173+
Assert.assertEquals(new BigDecimal("1.22"), body.get("amount"));
174+
175+
body = fileReader.readRow(HashMap.class);
176+
Assert.assertEquals(null, body.get("seq"));
177+
Assert.assertEquals("505", body.get("instSeq"));
178+
Assert.assertEquals("2016-02-03 12:22:33", DateUtil.format((Date) body.get("gmtApply"), "yyyy-MM-dd HH:mm:ss"));
179+
Assert.assertEquals(new BigDecimal("12"), body.get("applyNumber"));
180+
Assert.assertEquals(null, body.get("amount"));
181+
Assert.assertEquals(66, body.get("age"));
182+
Assert.assertEquals(125L, body.get("longN"));
183+
Assert.assertEquals(false, body.get("bol"));
184+
Assert.assertEquals("memo2", body.get("memo"));
185+
186+
fileReader.close();
187+
}
188+
189+
@After
190+
public void after() {
191+
tf.delete();
192+
}
193+
}

‎rdf-file-core/src/test/resources/codec/index/template/template1.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"head":[
3+
"totalCount|总笔数|Required|Long",
4+
"totalAmount|总金额|BigDecimal|Required"
5+
],
6+
"body":[
7+
"seq|流水号",
8+
"instSeq|基金公司订单号|Required",
9+
"gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss",
10+
"date|普通日期|Date:yyyyMMdd",
11+
"dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss",
12+
"applyNumber|普通数字|BigDecimal",
13+
"amount|金额|BigDecimal",
14+
"age|年龄|Integer",
15+
"longN|长整型|Long",
16+
"bol|布尔值|Boolean",
17+
"memo|备注"
18+
],
19+
"protocol":"DE",
20+
"rowCodecMode":"kv"
21+
}

‎rdf-file-core/src/test/resources/codec/index/template/template2.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"head":[
3+
"totalCount|总笔数|Required|Long",
4+
"totalAmount|总金额|BigDecimal|Required"
5+
],
6+
"body":[
7+
"seq|流水号",
8+
"instSeq|基金公司订单号|Required",
9+
"gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss",
10+
"date|普通日期|Date:yyyyMMdd",
11+
"dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss",
12+
"applyNumber|普通数字|BigDecimal",
13+
"amount|金额|BigDecimal",
14+
"age|年龄|Integer",
15+
"longN|长整型|Long",
16+
"bol|布尔值|Boolean",
17+
"memo|备注|default:defaultMemo"
18+
],
19+
"protocol":"DE",
20+
"rowCodecMode":"index",
21+
"params": {
22+
"rowCodecIndexPlace":"start"
23+
}
24+
}

0 commit comments

Comments
 (0)