0%

FlinkSQL-cumulateWindow

SQL

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
-- 创建订单表连接器  
CREATE TABLE orders
(
order_id STRING,
user_id STRING,
amount DOUBLE,
order_time TIMESTAMP(3),
ptime as PROCTIME(),
rtime as CAST(order_time AS TIMESTAMP_LTZ(3)),
WATERMARK FOR rtime AS rtime - INTERVAL '5' SECOND
) WITH (
'connector' = 'filesystem',
'path' = 'file://${order.file.path}',
'format' = 'csv',
'csv.ignore-parse-errors' = 'true',
'csv.allow-comments' = 'true'
);
--
CREATE TABLE console1
(
user_id STRING,
date_date STRING,
date_time STRING,
msg_time STRING,
total_amount DOUBLE,
order_times BIGINT
)
with (
'connector' = 'console',
'connector.name' = 'csl1'
);
-- 统计一天内,各用户的交易总额和支付次数,10分钟一次输出
INSERT INTO console1
SELECT user_id
, DATE_FORMAT(window_start, 'yyyyMMdd') AS data_date
, DATE_FORMAT(window_start, 'HH:mm:ss') AS data_time
, DATE_FORMAT(window_end, 'yyyy-MM-dd HH:mm:ss') AS msg_time
, SUM(amount) AS total_amount
, count(distinct order_id) AS video_exp_cnt
FROM
TABLE(
CUMULATE(
TABLE orders
, DESCRIPTOR(
rtime
)
, INTERVAL '10' MINUTES
, INTERVAL '1' DAYS
, INTERVAL '0' HOURS
, TRUE
)
)
GROUP BY user_id
, window_start
, window_end

RelNodes

1
2
3
4
5
6
7
8
Sink(table=[default_catalog.default_database.console1], fields=[user_id, data_date, data_time, msg_time, total_amount, video_exp_cnt])
+- Calc(select=[user_id, DATE_FORMAT(window_start, _UTF-16LE'yyyyMMdd') AS data_date, DATE_FORMAT(window_start, _UTF-16LE'HH:mm:ss') AS data_time, DATE_FORMAT(window_end, _UTF-16LE'yyyy-MM-dd HH:mm:ss') AS msg_time, total_amount, video_exp_cnt])
+- GlobalWindowAggregate(groupBy=[user_id], window=[CUMULATE(slice_end=[$slice_end], max_size=[86400000 ms], step=[10 min], offset=[0 ms], allowLazyTriggering=[true])], select=[user_id, SUM(sum$0) AS total_amount, COUNT(distinct$0 count$1) AS video_exp_cnt, start('w$) AS window_start, end('w$) AS window_end])
+- Exchange(distribution=[hash[user_id]])
+- LocalWindowAggregate(groupBy=[user_id], window=[CUMULATE(time_col=[order_time], max_size=[86400000 ms], step=[10 min], offset=[0 ms], allowLazyTriggering=[true])], select=[user_id, SUM(amount) AS sum$0, COUNT(distinct$0 order_id) AS count$1, DISTINCT(order_id) AS distinct$0, slice_end('w$) AS $slice_end])
+- Calc(select=[user_id, amount, order_id, order_time])
+- WatermarkAssigner(rowtime=[order_time], watermark=[-(order_time, 5000:INTERVAL SECOND)])
+- TableSourceScan(table=[[default_catalog, default_database, orders]], fields=[order_id, user_id, amount, order_time])

Transformations

过程

Transformation Description
SourceTransformation [1]:TableSourceScan(table=[[default_catalog, default_database, orders]], fields=[order_id, user_id, amount, order_time])
OneInputTransformation [2]:Calc(select=[order_id, user_id, amount, order_time, PROCTIME() AS ptime, CAST(order_time AS TIMESTAMP_WITH_LOCAL_TIME_ZONE(3)) AS rtime])
OneInputTransformation [3]:WatermarkAssigner(rowtime=[rtime], watermark=[(rtime - 5000:INTERVAL SECOND)])
OneInputTransformation [4]:Calc(select=[user_id, amount, order_id, ptime])
PartitionTransformation [5]:Exchange(distribution=[hash[user_id]])
OneInputTransformation [6]:WindowAggregate(groupBy=[user_id], window=[CUMULATE(time_col=[ptime], max_size=[86400000 ms], step=[10 min], offset=[0 ms], allowLazyTriggering=[true])], select=[user_id, SUM(amount) AS total_amount, COUNT(DISTINCT order_id) AS video_exp_cnt, start('w$) AS window_start, end('w$) AS window_end])
OneInputTransformation [7]:Calc(select=[user_id, DATE_FORMAT(window_start, 'yyyyMMdd') AS data_date, DATE_FORMAT(window_start, 'HH:mm:ss') AS data_time, DATE_FORMAT(window_end, 'yyyy-MM-dd HH:mm:ss') AS msg_time, total_amount, video_exp_cnt])
LegacySinkTransformation

new Source API

  • 问题1: SQL -> Operation -> Transformation

Codegens

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
Source: orders[1] -> WatermarkAssigner[2] -> Calc[3] -> LocalWindowAggregate[4] (7/8)#0 - The compiled code for StreamExecCalc$11:   
/* 1 */
/* 2 */ public class StreamExecCalc$11 extends org.apache.flink.table.runtime.operators.TableStreamOperator
/* 3 */ implements org.apache.flink.streaming.api.operators.OneInputStreamOperator {
/* 4 */
/* 5 */ private final Object[] references;
/* 6 */ private transient org.apache.flink.table.runtime.typeutils.StringDataSerializer typeSerializer$5;
/* 7 */ org.apache.flink.table.data.BoxedWrapperRowData out = new org.apache.flink.table.data.BoxedWrapperRowData(4);
/* 8 */ private final org.apache.flink.streaming.runtime.streamrecord.StreamRecord outElement = new org.apache.flink.streaming.runtime.streamrecord.StreamRecord(null);
/* 9 */
/* 10 */ public StreamExecCalc$11(
/* 11 */ Object[] references,
/* 12 */ org.apache.flink.streaming.runtime.tasks.StreamTask task,
/* 13 */ org.apache.flink.streaming.api.graph.StreamConfig config,
/* 14 */ org.apache.flink.streaming.api.operators.Output output,
/* 15 */ org.apache.flink.streaming.runtime.tasks.ProcessingTimeService processingTimeService) throws Exception {
/* 16 */ this.references = references;
/* 17 */ typeSerializer$5 = (((org.apache.flink.table.runtime.typeutils.StringDataSerializer) references[0]));
/* 18 */ this.setup(task, config, output);
/* 19 */ if (this instanceof org.apache.flink.streaming.api.operators.AbstractStreamOperator) {
/* 20 */ ((org.apache.flink.streaming.api.operators.AbstractStreamOperator) this)
/* 21 */ .setProcessingTimeService(processingTimeService);
/* 22 */ }
/* 23 */ }
/* 24 */
/* 25 */ @Override
/* 26 */ public void open() throws Exception {
/* 27 */ super.open();
/* 28 */
/* 29 */ }
/* 30 */
/* 31 */ @Override
/* 32 */ public void processElement(org.apache.flink.streaming.runtime.streamrecord.StreamRecord element) throws Exception {
/* 33 */ org.apache.flink.table.data.RowData in1 = (org.apache.flink.table.data.RowData) element.getValue();
/* 34 */
/* 35 */ org.apache.flink.table.data.binary.BinaryStringData field$4;
/* 36 */ boolean isNull$4;
/* 37 */ org.apache.flink.table.data.binary.BinaryStringData field$6;
/* 38 */ double field$7;
/* 39 */ boolean isNull$7;
/* 40 */ org.apache.flink.table.data.binary.BinaryStringData field$8;
/* 41 */ boolean isNull$8;
/* 42 */ org.apache.flink.table.data.binary.BinaryStringData field$9;
/* 43 */ org.apache.flink.table.data.TimestampData field$10;
/* 44 */ boolean isNull$10;
/* 45 */
/* 46 */
/* 47 */
/* 48 */ isNull$8 = in1.isNullAt(0);
/* 49 */ field$8 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8;
/* 50 */ if (!isNull$8) {
/* 51 */ field$8 = ((org.apache.flink.table.data.binary.BinaryStringData) in1.getString(0));
/* 52 */ }
/* 53 */ field$9 = field$8;
/* 54 */ if (!isNull$8) {
/* 55 */ field$9 = (org.apache.flink.table.data.binary.BinaryStringData) (typeSerializer$5.copy(field$9));
/* 56 */ }
/* 57 */
/* 58 */ isNull$7 = in1.isNullAt(2);
/* 59 */ field$7 = -1.0d;
/* 60 */ if (!isNull$7) {
/* 61 */ field$7 = in1.getDouble(2);
/* 62 */ }
/* 63 */
/* 64 */ isNull$4 = in1.isNullAt(1);
/* 65 */ field$4 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8;
/* 66 */ if (!isNull$4) {
/* 67 */ field$4 = ((org.apache.flink.table.data.binary.BinaryStringData) in1.getString(1));
/* 68 */ }
/* 69 */ field$6 = field$4;
/* 70 */ if (!isNull$4) {
/* 71 */ field$6 = (org.apache.flink.table.data.binary.BinaryStringData) (typeSerializer$5.copy(field$6));
/* 72 */ }
/* 73 */
/* 74 */ isNull$10 = in1.isNullAt(3);
/* 75 */ field$10 = null;
/* 76 */ if (!isNull$10) {
/* 77 */ field$10 = in1.getTimestamp(3, 3);
/* 78 */ }
/* 79 */
/* 80 */ out.setRowKind(in1.getRowKind());
/* 81 */
/* 82 */
/* 83 */
/* 84 */ /*+ [CODE HINT START] ROW_FIELD_0 */
/* 85 */
/* 86 */ if (isNull$4) {
/* 87 */ out.setNullAt(0);
/* 88 */ } else {
/* 89 */ out.setNonPrimitiveValue(0, field$6);
/* 90 */ }
/* 91 */ /*+ [CODE HINT END] ROW_FIELD_0 */
/* 92 */
/* 93 */
/* 94 */ /*+ [CODE HINT START] ROW_FIELD_1 */
/* 95 */
/* 96 */ if (isNull$7) {
/* 97 */ out.setNullAt(1);
/* 98 */ } else {
/* 99 */ out.setDouble(1, field$7);
/* 100 */ }
/* 101 */ /*+ [CODE HINT END] ROW_FIELD_1 */
/* 102 */
/* 103 */
/* 104 */ /*+ [CODE HINT START] ROW_FIELD_2 */
/* 105 */
/* 106 */ if (isNull$8) {
/* 107 */ out.setNullAt(2);
/* 108 */ } else {
/* 109 */ out.setNonPrimitiveValue(2, field$9);
/* 110 */ }
/* 111 */ /*+ [CODE HINT END] ROW_FIELD_2 */
/* 112 */
/* 113 */
/* 114 */ /*+ [CODE HINT START] ROW_FIELD_3 */
/* 115 */
/* 116 */ if (isNull$10) {
/* 117 */ out.setNullAt(3);
/* 118 */ } else {
/* 119 */ out.setNonPrimitiveValue(3, field$10);
/* 120 */ }
/* 121 */ /*+ [CODE HINT END] ROW_FIELD_3 */
/* 122 */
/* 123 */
/* 124 */ output.collect(outElement.replace(out));
/* 125 */
/* 126 */
/* 127 */ }
/* 128 */
/* 129 */
/* 130 */
/* 131 */ @Override
/* 132 */ public void close() throws Exception {
/* 133 */ super.close();
/* 134 */
/* 135 */ }
/* 136 */
/* 137 */
/* 138 */ }
/* 139 */