@@ -18,12 +18,20 @@ Licensed to the Apache Software Foundation (ASF) under one or more
18
18
19
19
package org .apache .flume .sink .gbase ;
20
20
21
+ import java .net .InetAddress ;
22
+ import java .net .UnknownHostException ;
23
+ import java .sql .Connection ;
24
+ import java .sql .DriverManager ;
25
+ import java .sql .SQLException ;
26
+ import java .sql .Statement ;
27
+
21
28
import org .apache .flume .Channel ;
22
29
import org .apache .flume .Context ;
23
30
import org .apache .flume .CounterGroup ;
24
31
import org .apache .flume .EventDeliveryException ;
25
32
import org .apache .flume .conf .Configurable ;
26
33
import org .apache .flume .sink .AbstractSink ;
34
+ import org .apache .flume .source .http .HTTPSourceConfigurationConstants ;
27
35
import org .slf4j .Logger ;
28
36
import org .slf4j .LoggerFactory ;
29
37
@@ -39,9 +47,16 @@ public class GBase8aSink extends AbstractSink implements Configurable {
39
47
private static final Logger logger = LoggerFactory .getLogger (GBase8aSink .class );
40
48
41
49
private PassiveHttpSink httpSink ;
50
+ private String connectUrl ;
51
+ private String driverClassName ;
52
+ private String userName ;
53
+ private String passWord ;
54
+ private String loadSql ;
55
+ private int loadInterval = 0 ;
56
+ private Connection conn = null ;
57
+ private Statement stm = null ;
42
58
43
59
private CounterGroup counterGroup ;
44
- private int batchSize ;
45
60
46
61
public GBase8aSink () {
47
62
counterGroup = new CounterGroup ();
@@ -52,53 +67,135 @@ public GBase8aSink() {
52
67
public void configure (Context context ) {
53
68
httpSink .configure (context );
54
69
55
- batchSize = context .getInteger (GBase8aSinkConstants .BATCH_SIZE ,
56
- GBase8aSinkConstants .DFLT_BATCH_SIZE );
57
- logger .debug (this .getName () + " " + "batch size set to " + String .valueOf (batchSize ));
58
- Preconditions .checkArgument (batchSize > 0 , "Batch size must be > 0" );
70
+ connectUrl = context .getString (GBase8aSinkConstants .CONNECTION_STRING );
71
+ Preconditions .checkArgument (connectUrl != null && connectUrl .trim ().length () != 0 ,
72
+ "connect url must be a string" );
73
+
74
+ userName = context .getString (GBase8aSinkConstants .CONNECTION_USERNAME );
75
+ passWord = context .getString (GBase8aSinkConstants .CONNECTION_PASSWORD );
76
+ driverClassName = context .getString (GBase8aSinkConstants .CONNECTION_DRIVER_CLASS ,
77
+ GBase8aSinkConstants .DFLT_DRIVER_CLASS );
78
+
79
+ loadInterval = context .getInteger (GBase8aSinkConstants .LOAD_INTERVAL ,
80
+ GBase8aSinkConstants .DFLT_LOAD_INTERVAL );
81
+ Preconditions .checkArgument (loadInterval >= 0 && loadInterval < 30 ,
82
+ "loadInterval must be in [0,30)." );
83
+
84
+ loadSql = context .getString (GBase8aSinkConstants .SQL_STRING );
85
+ Preconditions .checkArgument (loadSql != null && loadSql .trim ().length () != 0 ,
86
+ "load sql must be a string" );
87
+ try {
88
+ Integer port = context .getInteger (HTTPSourceConfigurationConstants .CONFIG_PORT );
89
+ /* 获取本机hostname对应的IP */
90
+ String IP = InetAddress .getLocalHost ().getHostAddress ();
91
+ logger .info ("local ip : {}, port : {}" , IP , port );
92
+ loadSql = loadSql .replaceAll ("\\ $\\ {localhost\\ }" , IP + ":" + port );
93
+ } catch (UnknownHostException e ) {
94
+ logger .error ("Error while get local IP. Exception follows." , e );
95
+ }
59
96
}
60
97
61
98
@ Override
62
99
public Status process () throws EventDeliveryException {
63
100
Status status = Status .READY ;
64
-
65
- /*
66
- * TODO 等待合适时间通过 jdbc 通知 8a 来读取数据
67
- */
101
+
102
+ try {
103
+ stm .execute (loadSql );
104
+ if (stm .getUpdateCount () == 0 ) {
105
+ Thread .sleep (loadInterval * 1000 );
106
+ }
107
+ } catch (Exception e ) {
108
+ logger .error ("Error while execute GBase8a. Exception follows." , e );
109
+ reconnectGBase8a ();
110
+ status = Status .BACKOFF ;
111
+ }
68
112
69
113
return status ;
70
114
}
71
115
116
+ /*
117
+ * (non-Javadoc)
118
+ *
119
+ * @see java.lang.Object#toString()
120
+ */
121
+ @ Override
122
+ public String toString () {
123
+ return "GBase8aSink [httpSink=" + httpSink + ", connectUrl=" + connectUrl + ", driverClassName="
124
+ + driverClassName + ", userName=" + userName + ", passWord=" + passWord + ", loadSql="
125
+ + loadSql + ", conn=" + conn + ", stm=" + stm + ", counterGroup=" + counterGroup + "]" ;
126
+ }
127
+
128
+ /**
129
+ * connect to GBase 8a MPP cluster.
130
+ *
131
+ * @author chensj
132
+ */
133
+ private void connectGBase8a () {
134
+ try {
135
+ Class .forName (driverClassName );
136
+ conn = DriverManager .getConnection (connectUrl , userName , passWord );
137
+ stm = conn .createStatement ();
138
+ logger .info ("connected GBase8a." );
139
+ } catch (ClassNotFoundException e ) {
140
+ logger .error ("Error while Connecting GBase8a. Exception follows." , e );
141
+ } catch (SQLException e ) {
142
+ logger .error ("Error while getConnection GBase8a. Exception follows." , e );
143
+ }
144
+ }
145
+
146
+ private void reconnectGBase8a () {
147
+ disconnectGBase8a ();
148
+ connectGBase8a ();
149
+ }
150
+
72
151
@ Override
73
152
public void start () {
74
153
logger .info ("Starting {}..." , this );
75
154
76
155
counterGroup .setName (this .getName ());
77
- super . start ();
156
+ connectGBase8a ();
78
157
httpSink .start ();
79
-
158
+ super . start ();
80
159
logger .info ("GBase 8a sink {} started." , getName ());
81
160
}
82
161
83
162
@ Override
84
163
public void stop () {
85
164
logger .info ("GBase 8a sink {} stopping..." , getName ());
165
+ disconnectGBase8a ();
86
166
87
167
httpSink .stop ();
88
168
super .stop ();
89
169
90
170
logger .info ("GBase 8a sink {} stopped. Event metrics: {}" , getName (), counterGroup );
91
171
}
92
172
93
- @ Override
94
- public String toString () {
95
- return "GBase8a " + getName () + " { batchSize: " + batchSize + " }" ;
173
+ /**
174
+ *
175
+ */
176
+ private void disconnectGBase8a () {
177
+ if (stm != null ) {
178
+ try {
179
+ if (!stm .isClosed ()) {
180
+ stm .close ();
181
+ }
182
+ } catch (Exception e ) {
183
+ logger .error ("Error while close stm. Exception follows." , e );
184
+ }
185
+ if (conn != null ) {
186
+ try {
187
+ conn .close ();
188
+ } catch (SQLException e ) {
189
+ logger .error ("Error while close connect. Exception follows." , e );
190
+ }
191
+ }
192
+ }
193
+
96
194
}
97
195
98
196
@ Override
99
197
public synchronized void setChannel (Channel channel ) {
100
198
httpSink .setChannel (channel );
101
199
super .setChannel (channel );
102
200
}
103
-
104
201
}
0 commit comments