1 """
2 The LDBDWClient module provides an API for connecting to and making requests of a LDBDWServer.
3
4 This file is part of the Grid LSC User Environment (GLUE)
5
6 GLUE is free software: you can redistribute it and/or modify it under the
7 terms of the GNU General Public License as published by the Free Software
8 Foundation, either version 3 of the License, or (at your option) any later
9 version.
10
11 This program is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
14 details.
15
16 You should have received a copy of the GNU General Public License along with
17 this program. If not, see <http://www.gnu.org/licenses/>.
18 """
19
20 from glue import git_version
21 __date__ = git_version.date
22 __version__ = git_version.id
23
24 import sys
25 import os
26 import types
27 import re
28 import six.moves.cPickle
29 import xml.parsers.expat
30 import six.moves.http_client
31
32 try:
33 from cjson import (decode, encode)
34 except ImportError:
35 from json import (loads as decode, dumps as encode)
36
37 try:
38 import M2Crypto
39 except ImportError as e:
40 sys.stderr.write("""
41 ligo_data_find requires M2Crypto
42
43 On CentOS 5 and other RHEL based platforms
44 this package is available from the EPEL
45 repository by doing
46
47 yum install m2crypto
48
49 For Debian Lenny this package is available
50 by doing
51
52 apt-get install python-m2crypto
53
54 Mac OS X users can find this package in
55 MacPorts.
56
57 %s
58 """ % e)
59 sys.exit(1)
60
61
64
65
67 """
68 A very simple LIGO_LW XML parser class that reads the only keeps
69 tables that do not contain the strings sngl_ or multi_
70
71 The class is not very robust as can have problems if the line
72 breaks do not appear in the standard places in the XML file.
73 """
75 """
76 Constructs an instance.
77
78 The private variable ignore_pat determines what tables we ignore.
79 """
80 self.__p = xml.parsers.expat.ParserCreate()
81 self.__in_table = 0
82 self.__silent = 0
83 self.__ignore_pat = re.compile(r'.*(sngl_|multi_).*', re.IGNORECASE)
84 self.__p.StartElementHandler = self.start_element
85 self.__p.EndElementHandler = self.end_element
86
88 """
89 Destroys an instance by shutting down and deleting the parser.
90 """
91 self.__p("",1)
92 del self.__p
93
95 """
96 Callback for start of an XML element. Checks to see if we are
97 about to start a table that matches the ignore pattern.
98
99 @param name: the name of the tag being opened
100 @type name: string
101
102 @param attrs: a dictionary of the attributes for the tag being opened
103 @type attrs: dictionary
104 """
105 if name.lower() == "table":
106 for attr in attrs.keys():
107 if attr.lower() == "name":
108 if self.__ignore_pat.search(attrs[attr]):
109 self.__in_table = 1
110
112 """
113 Callback for the end of an XML element. If the ignore flag is
114 set, reset it so we start outputing the table again.
115
116 @param name: the name of the tag being closed
117 @type name: string
118 """
119 if name.lower() == "table":
120 if self.__in_table:
121 self.__in_table = 0
122
124 """
125 For each line we are passed, call the XML parser. Returns the
126 line if we are outside one of the ignored tables, otherwise
127 returns the empty string.
128
129 @param line: the line of the LIGO_LW XML file to be parsed
130 @type line: string
131
132 @return: the line of XML passed in or the null string
133 @rtype: string
134 """
135 self.__p.Parse(line)
136 if self.__in_table:
137 self.__silent = 1
138 if not self.__silent:
139 ret = line
140 else:
141 ret = ""
142 if not self.__in_table:
143 self.__silent = 0
144 return ret
145
147 """
148 Follow the usual path that GSI libraries would
149 follow to find a valid proxy credential but
150 also allow an end entity certificate to be used
151 along with an unencrypted private key if they
152 are pointed to by X509_USER_CERT and X509_USER_KEY
153 since we expect this will be the output from
154 the eventual ligo-login wrapper around
155 kinit and then myproxy-login.
156 """
157
158
159 if 'X509_USER_PROXY' in os.environ:
160 filePath = os.environ['X509_USER_PROXY']
161 if validateProxy(filePath):
162 return filePath, filePath
163 else:
164 RFCproxyUsage()
165 sys.exit(1)
166
167
168 if 'X509_USER_CERT' in os.environ:
169 if 'X509_USER_KEY' in os.environ:
170 certFile = os.environ['X509_USER_CERT']
171 keyFile = os.environ['X509_USER_KEY']
172 return certFile, keyFile
173
174
175 uid = os.getuid()
176 path = "/tmp/x509up_u%d" % uid
177
178 if os.access(path, os.R_OK):
179 if validateProxy(path):
180 return path, path
181 else:
182 RFCproxyUsage()
183 sys.exit(1)
184
185
186 RFCproxyUsage()
187 sys.exit(1)
188
190 """
191 Test that the proxy certificate is RFC 3820
192 compliant and that it is valid for at least
193 the next 15 minutes.
194 """
195
196
197 try:
198 proxy = M2Crypto.X509.load_cert(path)
199 except Exception as e:
200 msg = "Unable to load proxy from path %s : %s\n" % (path, e)
201 sys.stderr.write(msg)
202 sys.exit(1)
203
204
205
206 try:
207 proxy.get_ext("proxyCertInfo")
208 except LookupError:
209
210
211 subject = proxy.get_subject().as_text()
212 if re.search(r'.+CN=proxy$', subject):
213
214 RFCproxyUsage()
215 sys.exit(1)
216
217
218 try:
219 expireASN1 = proxy.get_not_after().__str__()
220 expireGMT = time.strptime(expireASN1, "%b %d %H:%M:%S %Y %Z")
221 expireUTC = calendar.timegm(expireGMT)
222 now = int(time.time())
223 secondsLeft = expireUTC - now
224 except Exception as e:
225
226
227 secondsLeft = 3600
228
229 if secondsLeft <= 0:
230 msg = """\
231 Your proxy certificate is expired.
232
233 Please generate a new proxy certificate and
234 try again.
235 """
236 sys.stderr.write(msg)
237 sys.exit(1)
238
239 if secondsLeft < (60 * 15):
240 msg = """\
241 Your proxy certificate expires in less than
242 15 minutes.
243
244 Please generate a new proxy certificate and
245 try again.
246 """
247 sys.stderr.write(msg)
248 sys.exit(1)
249
250
251 return True
252
253
255 """
256 Print a simple error message about not finding
257 a RFC 3820 compliant proxy certificate.
258 """
259 msg = """\
260 Could not find a valid proxy credential.
261 LIGO users, please run 'ligo-proxy-init' and try again.
262 Others, please run 'grid-proxy-init' and try again.
263 """
264
265 sys.stderr.write(msg)
266
267
269 """Exceptions returned by server"""
270 pass
271
272
274 - def __init__(self, host, port = None, protocol = None, identity = None):
275 """
276 """
277 self.host = host
278 self.port = port
279 self.protocol = protocol
280
281 if self.port:
282 self.server = "%s:%d" % (self.host, self.port)
283 else:
284 self.server = self.host
285
286
287
288
289 self.identity = identity
290
291
292 if protocol == "http":
293 self.certFile = None
294 self.keyFile = None
295 else:
296 self.certFile, self.keyFile = findCredential()
297
299 """
300 """
301 server = self.server
302 protocol = self.protocol
303
304 if protocol == "https":
305
306 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile)
307 else:
308 h = six.moves.http_client.HTTPConnection(server)
309
310 url = "/ldbd/ping.json"
311 headers = {"Content-type" : "application/json"}
312 data = ""
313 body = encode(protocol)
314
315 try:
316 h.request("POST", url, body, headers)
317 response = h.getresponse()
318 except Exception as e:
319 msg = "Error pinging server %s: %s" % (server, e)
320 raise LDBDClientException(msg)
321
322 if response.status != 200:
323 msg = "Server returned code %d: %s" % (response.status, response.reason)
324 body = response.read()
325 msg += body
326 raise LDBDClientException(msg)
327
328
329 body = response.read()
330 msg = decode(body)
331
332 return msg
333
335 """
336 """
337
338 server = self.server
339 protocol = self.protocol
340 if protocol == "https":
341 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile)
342 else:
343 h = six.moves.http_client.HTTPConnection(server)
344
345 url = "/ldbd/query.json"
346 headers = {"Content-type" : "application/json"}
347 body = encode(protocol + ":" + sql)
348
349 try:
350 h.request("POST", url, body, headers)
351 response = h.getresponse()
352 except Exception as e:
353 msg = "Error querying server %s: %s" % (server, e)
354 raise LDBDClientException(msg)
355
356 if response.status != 200:
357 msg = "Server returned code %d: %s" % (response.status, response.reason)
358 body = response.read()
359 msg += body
360 raise LDBDClientException(msg)
361
362
363 body = response.read()
364 msg = decode(body)
365
366 return msg
367
369 """
370 """
371
372 server = self.server
373 protocol = self.protocol
374 if protocol == "https":
375 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile)
376 else:
377 msg = "Insecure connection DOES NOT surpport INSERT."
378 msg += '\nTo INSERT, authorized users please specify protocol "https" in your --segment-url argument.'
379 msg += '\nFor example, "--segment-url https://segdb.ligo.caltech.edu".'
380 raise LDBDClientException(msg)
381
382 url = "/ldbd/insert.json"
383 headers = {"Content-type" : "application/json"}
384 body = encode(xmltext)
385
386 try:
387 h.request("POST", url, body, headers)
388 response = h.getresponse()
389 except Exception as e:
390 msg = "Error querying server %s: %s" % (server, e)
391 raise LDBDClientException(msg)
392
393 if response.status != 200:
394 msg = "Server returned code %d: %s" % (response.status, response.reason)
395 body = response.read()
396 msg += body
397 raise LDBDClientException(msg)
398
399
400 body = response.read()
401 msg = decode(body)
402
403 return msg
404
406 """
407 """
408 server = self.server
409 protocol = self.protocol
410 if protocol == "https":
411 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile)
412 else:
413 msg = "Insecure connection DOES NOT surpport INSERTMAP."
414 msg += '\nTo INSERTMAP, authorized users please specify protocol "https" in your --segment-url argument.'
415 msg += '\nFor example, "--segment-url https://segdb.ligo.caltech.edu".'
416 raise LDBDClientException(msg)
417
418 url = "/ldbd/insertmap.json"
419 headers = {"Content-type" : "application/json"}
420
421 pmsg = six.moves.cPickle.dumps(lfnpfn_dict)
422 data = [xmltext, pmsg]
423 body = encode(data)
424
425 try:
426 h.request("POST", url, body, headers)
427 response = h.getresponse()
428 except Exception as e:
429 msg = "Error querying server %s: %s" % (server, e)
430 raise LDBDClientException(msg)
431
432 if response.status != 200:
433 msg = "Server returned code %d: %s" % (response.status, response.reason)
434 body = response.read()
435 msg += body
436 raise LDBDClientException(msg)
437
438
439 body = response.read()
440 msg = decode(body)
441
442 return msg
443
444
446 """
447 """
448 server = self.server
449 protocol = self.protocol
450 if protocol == "https":
451 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile)
452 else:
453 msg = "Insecure connection DOES NOT surpport INSERTDMT."
454 msg += '\nTo INSERTDMT, authorized users please specify protocol "https" in your --segment-url argument.'
455 msg += '\nFor example, "--segment-url https://segdb.ligo.caltech.edu".'
456 raise LDBDClientException(msg)
457
458 url = "/ldbd/insertdmt.json"
459 headers = {"Content-type" : "application/json"}
460 body = encode(xmltext)
461
462 try:
463 h.request("POST", url, body, headers)
464 response = h.getresponse()
465 except Exception as e:
466 msg = "Error querying server %s: %s" % (server, e)
467 raise LDBDClientException(msg)
468
469 if response.status != 200:
470 msg = "Server returned code %d: %s" % (response.status, response.reason)
471 body = response.read()
472 msg += body
473 raise LDBDClientException(msg)
474
475
476 body = response.read()
477 msg = decode(body)
478
479 return msg