Skip to content

Commit c195e55

Browse files
quhongweiquhongwei
authored
Merge pull request #51 from alipay/20191129-functionext
20191129 functionext
2 parents 12afc0f + 998adc2 commit c195e55

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1536
-85
lines changed

‎pom.xml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@
1111
<url>https://github.com/alipay/rdf-file</url>
1212

1313
<properties>
14-
<rdf.file.core.version>2.2.5</rdf.file.core.version>
15-
<rdf.file.oss.version>2.2.5</rdf.file.oss.version>
16-
<rdf.file.sftp.version>2.2.5</rdf.file.sftp.version>
14+
<rdf.file.core.version>2.2.6</rdf.file.core.version>
15+
<rdf.file.oss.version>2.2.6</rdf.file.oss.version>
16+
<rdf.file.sftp.version>2.2.6</rdf.file.sftp.version>
1717
</properties>
1818

1919
<modules>

‎rdf-file-core/src/main/java/com/alipay/rdf/file/codec/AbstractColumnInfoCodec.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/**
2+
* Alipay.com Inc.
3+
* Copyright (c) 2004-2019 All Rights Reserved.
4+
*/
5+
package com.alipay.rdf.file.codec;
6+
7+
import com.alipay.rdf.file.exception.RdfErrorEnum;
8+
import com.alipay.rdf.file.exception.RdfFileException;
9+
import com.alipay.rdf.file.loader.TemplateLoader;
10+
import com.alipay.rdf.file.meta.FileColumnMeta;
11+
import com.alipay.rdf.file.meta.FileMeta;
12+
import com.alipay.rdf.file.model.FileConfig;
13+
import com.alipay.rdf.file.model.FileDataTypeEnum;
14+
15+
import java.util.List;
16+
17+
/**
18+
* Copyright (C) 2013-2018 Ant Financial Services Group
19+
*
20+
* @author quhongwei
21+
* @version : AbstractColumnInfoCodec.java, v 0.1 2019年11月29日 12:55 quhongwei Exp $
22+
*/
23+
public abstract class AbstractColumnInfoCodec {
24+
25+
protected static String getValue(FileColumnMeta colMeta, String method) {
26+
String value;
27+
if ("desc".equalsIgnoreCase(method)) {
28+
value = colMeta.getDesc();
29+
} else if ("name".equalsIgnoreCase(method)) {
30+
value = colMeta.getName();
31+
} else {
32+
throw new RdfFileException(
33+
"rdf-file#AbstractColumnInfoCodec.getValue 无效方法参数method=" + method,
34+
RdfErrorEnum.UNSUPPORTED_OPERATION);
35+
}
36+
return value;
37+
}
38+
39+
protected static List<FileColumnMeta> getColumnMetas(FileConfig config, FileDataTypeEnum dataType) {
40+
FileMeta fileMeta = TemplateLoader.load(config);
41+
switch (dataType) {
42+
case HEAD:
43+
return fileMeta.getHeadColumns();
44+
case BODY:
45+
return fileMeta.getBodyColumns();
46+
case TAIL:
47+
return fileMeta.getTailColumns();
48+
default:
49+
throw new RdfFileException(
50+
"rdf-file#AbstractColumnInfoCodec.getColumnMetas dateType=" + dataType.name(),
51+
RdfErrorEnum.UNSUPPORTED_OPERATION);
52+
}
53+
}
54+
55+
}

‎rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnHorizontalCodec.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
* @author hongwei.quhw
2424
* @version $Id: HeadColumnCodec.java, v 0.1 2017-1-3 下午5:50:00 hongwei.quhw Exp $
2525
*/
26+
@Deprecated
2627
public class BodyColumnHorizontalCodec {
2728

2829
/**

‎rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnVerticalCodec.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
* @author hongwei.quhw
1919
* @version $Id: HeadColumnCodec.java, v 0.1 2017-1-3 下午5:50:00 hongwei.quhw Exp $
2020
*/
21+
@Deprecated
2122
public class BodyColumnVerticalCodec {
2223
public static final BodyColumnVerticalCodec instance = new BodyColumnVerticalCodec();
2324

‎rdf-file-core/src/main/java/com/alipay/rdf/file/codec/ColumnInfoHorizontalCodec.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package com.alipay.rdf.file.codec;
2+
3+
import com.alipay.rdf.file.exception.RdfErrorEnum;
4+
import com.alipay.rdf.file.exception.RdfFileException;
5+
import com.alipay.rdf.file.interfaces.FileReader;
6+
import com.alipay.rdf.file.interfaces.FileWriter;
7+
import com.alipay.rdf.file.loader.ProtocolLoader;
8+
import com.alipay.rdf.file.loader.TemplateLoader;
9+
import com.alipay.rdf.file.meta.FileColumnMeta;
10+
import com.alipay.rdf.file.meta.FileMeta;
11+
import com.alipay.rdf.file.model.FileConfig;
12+
import com.alipay.rdf.file.model.FileDataTypeEnum;
13+
import com.alipay.rdf.file.spi.RdfFileRowSplitSpi.SplitContext;
14+
import com.alipay.rdf.file.util.RdfFileUtil;
15+
16+
import java.util.List;
17+
18+
/**
19+
* Copyright (C) 2013-2018 Ant Financial Services Group
20+
*
21+
* 字段信息水平编码解码
22+
*
23+
* @author hongwei.quhw
24+
* @version $Id: ColumnInfoHorizontalCodec.java, v 0.1 2017-1-3 下午5:50:00 hongwei.quhw Exp $
25+
*/
26+
public class ColumnInfoHorizontalCodec extends AbstractColumnInfoCodec {
27+
28+
public static void serialize(FileDataTypeEnum layoutType, FileDataTypeEnum dataType, FileConfig config, FileWriter writer,
29+
String method) {
30+
FileMeta fileMeta = TemplateLoader.load(config);
31+
List<FileColumnMeta> colMetas = getColumnMetas(config, dataType);
32+
StringBuilder colHead = new StringBuilder();
33+
34+
if (null != RdfFileUtil.getRowSplit(config) && fileMeta.isStartWithSplit(layoutType)) {
35+
colHead.append(RdfFileUtil.getRowSplit(config));
36+
}
37+
38+
for (int i = 0; i < colMetas.size(); i++) {
39+
FileColumnMeta colMeta = colMetas.get(i);
40+
colHead.append(getValue(colMeta, method));
41+
//添加字段分割符
42+
if (null != RdfFileUtil.getRowSplit(config) && (i < colMetas.size() - 1 || fileMeta.isEndWithSplit(layoutType))) {
43+
colHead.append(RdfFileUtil.getRowSplit(config));
44+
}
45+
}
46+
writer.writeLine(colHead.toString());
47+
48+
}
49+
50+
public static <T> T deserialize(FileDataTypeEnum layoutType, FileDataTypeEnum dataType, FileConfig config,
51+
FileReader reader,
52+
String method) {
53+
String line = reader.readLine();
54+
RdfFileUtil.assertNotBlank(line, "文件=" + config.getFilePath() + ", " + layoutType.name() + " 内容缺失");
55+
56+
FileMeta fileMeta = TemplateLoader.load(config);
57+
String[] columns = ProtocolLoader.loadProtocol(fileMeta.getProtocol()).getRowSplit().split(
58+
new SplitContext(line, config, FileDataTypeEnum.BODY));
59+
List<FileColumnMeta> colMetas = getColumnMetas(config, dataType);
60+
61+
int splitLength = fileMeta.isStartWithSplit(layoutType) ? colMetas.size() + 1 : colMetas.size();
62+
splitLength = fileMeta.isEndWithSplit(layoutType) ? splitLength + 1 : splitLength;
63+
64+
if (splitLength != columns.length) {
65+
throw new RdfFileException("文件=" + config.getFilePath() + "， " + layoutType.name() + " line=" + line,
66+
RdfErrorEnum.DESERIALIZE_ERROR);
67+
}
68+
69+
int statIndex = fileMeta.isStartWithSplit(layoutType) ? 1 : 0;
70+
int endIndex = fileMeta.isEndWithSplit(layoutType) ? columns.length - 1 : columns.length;
71+
72+
for (int i = statIndex; i < endIndex; i++) {
73+
FileColumnMeta colMeta = colMetas.get(i - statIndex);
74+
if (!getValue(colMeta, method).equals(columns[i])) {
75+
throw new RdfFileException(
76+
"文件" + layoutType.name() + "字段校验：文件模板定义的第" + i + "个column为[" + colMetas.get(i).getDesc() + "], 实际文件中为["
77+
+ columns[i] + "]", RdfErrorEnum.DESERIALIZE_ERROR);
78+
}
79+
}
80+
return null;
81+
}
82+
83+
}

‎rdf-file-core/src/main/java/com/alipay/rdf/file/codec/ColumnInfoVerticalCodec.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package com.alipay.rdf.file.codec;
2+
3+
import com.alipay.rdf.file.exception.RdfErrorEnum;
4+
import com.alipay.rdf.file.exception.RdfFileException;
5+
import com.alipay.rdf.file.interfaces.FileReader;
6+
import com.alipay.rdf.file.interfaces.FileWriter;
7+
import com.alipay.rdf.file.meta.FileColumnMeta;
8+
import com.alipay.rdf.file.model.FileConfig;
9+
import com.alipay.rdf.file.model.FileDataTypeEnum;
10+
import com.alipay.rdf.file.util.RdfFileUtil;
11+
12+
/**
13+
* Copyright (C) 2013-2018 Ant Financial Services Group
14+
*
15+
* 字段信息纵向编码解码
16+
*
17+
* @author hongwei.quhw
18+
* @version $Id: ColumnInfoVerticalCodec.java, v 0.1 2017-1-3 下午5:50:00 hongwei.quhw Exp $
19+
*/
20+
public class ColumnInfoVerticalCodec extends AbstractColumnInfoCodec {
21+
public static final ColumnInfoVerticalCodec instance = new ColumnInfoVerticalCodec();
22+
23+
public static void serialize(FileDataTypeEnum layoutType, FileDataTypeEnum dataType, FileConfig fileConfig,
24+
FileWriter writer, String method) {
25+
//按行写入column字段
26+
for (FileColumnMeta colMeta : getColumnMetas(fileConfig, dataType)) {
27+
writer.writeLine(getValue(colMeta, method));
28+
}
29+
}
30+
31+
public static <T> T deserialize(FileDataTypeEnum layoutType, FileDataTypeEnum dataType, FileConfig fileConfig,
32+
FileReader reader, String method) {
33+
for (FileColumnMeta colMeta : getColumnMetas(fileConfig, dataType)) {
34+
String columName = RdfFileUtil.assertTrimNotBlank(reader.readLine());
35+
RdfFileUtil.assertNotBlank(columName, "文件=" + fileConfig.getFilePath() + ", " + layoutType.name() + " 内容缺失");
36+
37+
String tempalteValue = getValue(colMeta, method);
38+
if (!tempalteValue.equals(columName)) {
39+
throw new RdfFileException(
40+
"rdf-file#" + layoutType.name() + "模板中定义的column为" + tempalteValue + ", 文件中读取的column为" + columName + " 不一致",
41+
RdfErrorEnum.VALIDATE_ERROR);
42+
43+
}
44+
}
45+
return null;
46+
}
47+
}

‎rdf-file-core/src/main/java/com/alipay/rdf/file/codec/RowsCodec.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,6 @@ public static FileColumnMeta getColumnMeta(FileColumnMeta colMeta, FileMeta file
141141

142142
return new FileColumnMeta(colMeta.getColIndex(), colMeta.getName(), colMeta.getDesc(),
143143
colMeta.getType(), colMeta.isRequired(), colMeta.getRange(), colMeta.getDefaultValue(),
144-
fileMeta);
144+
fileMeta, colMeta.getDataType());
145145
}
146146
}

‎rdf-file-core/src/main/java/com/alipay/rdf/file/function/BodyColumnFunction.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,13 @@
1919
* Copyright (C) 2013-2018 Ant Financial Services Group
2020
*
2121
* body 字段
22-
*
22+
*
23+
* 使用更通用的ColumnInfoFunction函数功能替换
24+
*
2325
* @author hongwei.quhw
2426
* @version $Id: BodyColumnFunction.java, v 0.1 2017年8月19日 下午1:21:38 hongwei.quhw Exp $
2527
*/
28+
@Deprecated
2629
@SuppressWarnings("rawtypes")
2730
public class BodyColumnFunction extends RdfFunction {
2831
@Override

‎rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnFunctionWrapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ private String serialize(Object field, FileColumnMeta columnMeta, FileConfig fil
127127
}
128128

129129
private ColumnRegEx getColumnRegEx(FileColumnMeta columnMeta) {
130-
String key = columnMeta.getFileMeta().getTemplatePath() + "-" + columnMeta.getName();
130+
String key = columnMeta.getFileMeta().getTemplatePath() + "-" + columnMeta.getDataType().name() + "-" + columnMeta.getName();
131131
ColumnRegEx columnRegEx = columnRegExs.get(key);
132132
if (null == columnRegEx) {
133133
columnRegEx = new ColumnRegEx();

‎rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnInfoFunction.java‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package com.alipay.rdf.file.function;
2+
3+
import com.alipay.rdf.file.codec.ColumnInfoHorizontalCodec;
4+
import com.alipay.rdf.file.codec.ColumnInfoVerticalCodec;
5+
import com.alipay.rdf.file.exception.RdfErrorEnum;
6+
import com.alipay.rdf.file.exception.RdfFileException;
7+
import com.alipay.rdf.file.loader.ExtensionLoader;
8+
import com.alipay.rdf.file.loader.FormatLoader;
9+
import com.alipay.rdf.file.loader.TemplateLoader;
10+
import com.alipay.rdf.file.meta.FileMeta;
11+
import com.alipay.rdf.file.model.FileDataTypeEnum;
12+
import com.alipay.rdf.file.protocol.RowDefinition;
13+
import com.alipay.rdf.file.spi.RdfFileColumnTypeSpi;
14+
import com.alipay.rdf.file.spi.RdfFileFormatSpi;
15+
import com.alipay.rdf.file.util.RdfFileUtil;
16+
17+
/**
18+
* Copyright (C) 2013-2018 Ant Financial Services Group
19+
*
20+
* 字段信息序列化反序列化函数
21+
*
22+
* 参考fund.xml协议
23+
*
24+
* 1. ${columnInfo.count(body)} 计算body的字段数
25+
* 2. ${columnInfo.vertical(body,name)} 每个body字段名作为一行
26+
* 3. ${columnInfo.horizontal(tail,name)} 所有文件尾字段名作为一行数据
27+
*
28+
* 函数第一个参数代表对文件数据模板中哪个部分的字段进行信息处理
29+
* 函数第二个参数代表对文件数据模板中name或者desc字段处理
30+
*
31+
* @author quhongwei
32+
* @version : ColumnInfoFunction.java, v 0.1 2019年11月29日 15:50 quhongwei Exp $
33+
*/
34+
public class ColumnInfoFunction extends RdfFunction {
35+
@Override
36+
public void checkParams() {
37+
if (("horizontal".equals(expression) || "vertical".equals(expression)) && (null == params || params.length != 2)) {
38+
throw new RdfFileException("rdf-file#ColumnInfoFunction.checkParams() 指定的参数应该为两个", RdfErrorEnum.FUNCTION_ERROR);
39+
} else if ("count".equals(expression) && (null == params || params.length != 1)) {
40+
throw new RdfFileException("rdf-file#ColumnInfoFunction.checkParams() 指定的参数应该为一个", RdfErrorEnum.FUNCTION_ERROR);
41+
}
42+
}
43+
44+
@Override
45+
public int rowsAffected(RowDefinition rd, FileMeta fileMeta) {
46+
if ("horizontal".equals(expression)) {
47+
return 1;
48+
} else if ("vertical".equals(expression)) {
49+
return getColumnSize(fileMeta);
50+
} else if ("count".equals(expression)) {
51+
return 1;
52+
} else {
53+
throw new RdfFileException(
54+
"rdf-file#ColumnInfoFunction" + expression + ", 无法计算rowsAffected",
55+
RdfErrorEnum.UNSUPPORTED_OPERATION);
56+
}
57+
}
58+
59+
public void horizontal(FuncContext ctx) {
60+
FileDataTypeEnum dataType = FileDataTypeEnum.valueOf(params[0].toUpperCase());
61+
if (CodecType.SERIALIZE.equals(ctx.codecType)) {
62+
ColumnInfoHorizontalCodec.serialize(rowType, dataType, ctx.fileConfig, ctx.writer, params[1]);
63+
64+
} else if (CodecType.DESERIALIZE.equals(ctx.codecType)) {
65+
ColumnInfoHorizontalCodec.deserialize(rowType, dataType, ctx.fileConfig, ctx.reader, params[1]);
66+
}
67+
}
68+
69+
public void vertical(FuncContext ctx) {
70+
FileDataTypeEnum dataType = FileDataTypeEnum.valueOf(params[0].toUpperCase());
71+
72+
if (CodecType.SERIALIZE.equals(ctx.codecType)) {
73+
ColumnInfoVerticalCodec.serialize(rowType, dataType, ctx.fileConfig, ctx.writer, params[1]);
74+
} else if (CodecType.DESERIALIZE.equals(ctx.codecType)) {
75+
ColumnInfoVerticalCodec.deserialize(rowType, dataType, ctx.fileConfig, ctx.reader, params[1]);
76+
}
77+
}
78+
79+
public void count(FuncContext ctx) {
80+
FileMeta fileMeta = TemplateLoader.load(ctx.fileConfig);
81+
String typeName = ctx.columnMeta.getType().getName();
82+
83+
RdfFileFormatSpi columnFormat = FormatLoader.getColumnFormt(fileMeta.getProtocol(),
84+
typeName);
85+
RdfFileUtil.assertNotNull(columnFormat, "类型type=" + typeName + " 对应的format没有");
86+
RdfFileColumnTypeSpi columnTypeCodec = ExtensionLoader
87+
.getExtensionLoader(RdfFileColumnTypeSpi.class).getExtension(typeName);
88+
RdfFileUtil.assertNotNull(columnTypeCodec, "没有type=" + typeName + " 对应的类型codec");
89+
90+
boolean startWithSplit = null != RdfFileUtil.getRowSplit(ctx.fileConfig) && fileMeta.isStartWithSplit(rowType);
91+
boolean endtWithSplit = null != RdfFileUtil.getRowSplit(ctx.fileConfig) && fileMeta.isEndWithSplit(rowType);
92+
93+
switch (ctx.codecType) {
94+
case SERIALIZE:
95+
StringBuilder line = new StringBuilder();
96+
if (startWithSplit) {
97+
line.append(RdfFileUtil.getRowSplit(ctx.fileConfig));
98+
}
99+
line.append(getColumnSize(fileMeta));
100+
if (endtWithSplit) {
101+
line.append(RdfFileUtil.getRowSplit(ctx.fileConfig));
102+
}
103+
ctx.writer.writeLine(columnFormat.serialize(line.toString(), ctx.columnMeta, ctx.fileConfig));
104+
break;
105+
case DESERIALIZE:
106+
String value = ctx.reader.readLine();
107+
if (startWithSplit) {
108+
value = value.substring(RdfFileUtil.getRowSplit(ctx.fileConfig).length());
109+
}
110+
if (endtWithSplit) {
111+
value = value.substring(0, value.length() - RdfFileUtil.getRowSplit(ctx.fileConfig).length());
112+
}
113+
Object field = columnTypeCodec.deserialize(value, ctx.columnMeta);
114+
RdfFileUtil.assertEquals(field.toString(),
115+
String.valueOf(fileMeta.getBodyColumns().size()));
116+
break;
117+
default:
118+
throw new RdfFileException("不支持序列化反序列化类型" + ctx.codecType.name(),
119+
RdfErrorEnum.UNSUPPORTED_OPERATION);
120+
}
121+
}
122+
123+
private int getColumnSize(FileMeta fileMeta) {
124+
FileDataTypeEnum dataType = FileDataTypeEnum.valueOf(params[0].toUpperCase());
125+
switch (dataType) {
126+
case HEAD:
127+
return fileMeta.getHeadColumns().size();
128+
case BODY:
129+
return fileMeta.getBodyColumns().size();
130+
case TAIL:
131+
return fileMeta.getTailColumns().size();
132+
default:
133+
throw new RdfFileException(
134+
"rdf-file#ColumnInfoFunction.rowsAffected dateType=" + dataType.name(),
135+
RdfErrorEnum.UNSUPPORTED_OPERATION);
136+
}
137+
}
138+
139+
}

0 commit comments

Comments
 (0)