1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
| public abstract class CommonSinkOperator<T extends Serializable> extends AbstractStreamOperator<Object>
implements ProcessingTimeCallback,
OneInputStreamOperator<T, Object> {
private List<T> list;
private ListState<T> listState;
private int bathSize;
private long interval;
private ProcessingTimeService processingTimeService;
public CommonSinkOperator() {
}
public CommonSinkOperator(int batchSize, long interval)
{
this.chainingStrategy = ChainingStrategy.ALWAYS;
this.batchSize = batchSize;
this.interval = interval;
}
@Override
public void open() throws Exception {
super.open();
if (interval > 0 && batchSize > 1) {
processingTimeService = getProcessingTimeService();
long now = processingTimeService.getCurrentProcessingTime();
processingTimeService.registerTimer(now + interval, this);
}
}
@Override
public void initializeState(
StateInitializationContext context) throws Exception {
super.initializeState(context);
this.list = new ArrayList<T>();
listState = context.getOperatorStateStore()
.getSerializableListState("batch-interval-sink");
if (context.isRestored()) {
listState.get().forEach(x -> {
list.add(x);
});
}
}
@Override
public void processElement(
StreamRecord<T> element) throws Exception {
list.add(element.getValue());
if (list.size() >= batchSize) {
saveRecords(list);
}
}
@Override
public void snapshotState(
StateSnapshotContext context) throws Exception {
super.snapshotState(context);
if (list.size() > 0) {
listState.clear();
listState.addAll(list);
}
}
@Override
public void onProcessingTime(long timestamp)
throws Exception {
if (list.size() > 0) {
saveRecords(list);
list.clear();
}
long now =processingTimeService.getCurrentProcessingTime();
processingTimeService.registerTimer(now + interval,this);
}
public abstract void saveRecords(List<T> datas);
}
|